2024-11-12 18:14:35,001 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-12 18:14:35,039 main DEBUG Took 0.015765 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-12 18:14:35,040 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-12 18:14:35,040 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-12 18:14:35,041 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-12 18:14:35,043 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 18:14:35,053 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-12 18:14:35,068 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 18:14:35,070 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 18:14:35,071 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 18:14:35,071 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 18:14:35,072 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 18:14:35,072 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 18:14:35,074 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 18:14:35,074 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 18:14:35,075 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 18:14:35,077 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 18:14:35,078 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 18:14:35,079 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 18:14:35,079 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 18:14:35,080 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 18:14:35,080 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 18:14:35,081 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 18:14:35,081 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 18:14:35,081 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 18:14:35,082 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 18:14:35,082 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 18:14:35,083 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 18:14:35,083 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 18:14:35,084 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 18:14:35,088 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 18:14:35,089 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 18:14:35,090 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-12 18:14:35,092 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 18:14:35,093 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-12 18:14:35,095 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-12 18:14:35,096 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-12 18:14:35,101 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-12 18:14:35,101 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-12 18:14:35,113 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-12 18:14:35,116 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-12 18:14:35,119 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-12 18:14:35,119 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-12 18:14:35,120 main DEBUG createAppenders(={Console}) 2024-11-12 18:14:35,121 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-11-12 18:14:35,121 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-12 18:14:35,121 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-11-12 18:14:35,122 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-12 18:14:35,122 main DEBUG OutputStream closed 2024-11-12 18:14:35,123 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-12 18:14:35,123 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-12 18:14:35,123 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-11-12 18:14:35,225 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-12 18:14:35,228 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-12 18:14:35,232 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-12 18:14:35,233 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-12 18:14:35,234 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-12 18:14:35,234 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-12 18:14:35,234 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-12 18:14:35,235 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-12 18:14:35,235 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-12 18:14:35,236 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-12 18:14:35,236 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-12 18:14:35,237 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-12 18:14:35,237 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-12 18:14:35,237 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-12 18:14:35,238 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-12 18:14:35,238 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-12 18:14:35,238 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-12 18:14:35,239 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-12 18:14:35,242 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-12 18:14:35,243 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-11-12 18:14:35,243 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-12 18:14:35,244 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-11-12T18:14:35,263 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-11-12 18:14:35,267 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-12 18:14:35,268 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-12T18:14:35,663 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/085c9431-6410-182f-5cbd-7a8569c5232a 2024-11-12T18:14:35,698 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-12T18:14:35,729 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/085c9431-6410-182f-5cbd-7a8569c5232a/cluster_1655c978-3304-a2e8-86fb-5151dc4c47d4, deleteOnExit=true 2024-11-12T18:14:35,730 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-12T18:14:35,731 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/085c9431-6410-182f-5cbd-7a8569c5232a/test.cache.data in system properties and HBase conf 2024-11-12T18:14:35,732 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/085c9431-6410-182f-5cbd-7a8569c5232a/hadoop.tmp.dir in system properties and HBase conf 2024-11-12T18:14:35,733 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/085c9431-6410-182f-5cbd-7a8569c5232a/hadoop.log.dir in system properties and HBase conf 2024-11-12T18:14:35,733 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/085c9431-6410-182f-5cbd-7a8569c5232a/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-12T18:14:35,734 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/085c9431-6410-182f-5cbd-7a8569c5232a/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-12T18:14:35,735 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-12T18:14:35,960 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-12T18:14:36,099 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T18:14:36,106 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/085c9431-6410-182f-5cbd-7a8569c5232a/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-12T18:14:36,107 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/085c9431-6410-182f-5cbd-7a8569c5232a/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-12T18:14:36,107 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/085c9431-6410-182f-5cbd-7a8569c5232a/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-12T18:14:36,108 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/085c9431-6410-182f-5cbd-7a8569c5232a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-12T18:14:36,109 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/085c9431-6410-182f-5cbd-7a8569c5232a/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-12T18:14:36,109 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/085c9431-6410-182f-5cbd-7a8569c5232a/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-12T18:14:36,110 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/085c9431-6410-182f-5cbd-7a8569c5232a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-12T18:14:36,110 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/085c9431-6410-182f-5cbd-7a8569c5232a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-12T18:14:36,111 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/085c9431-6410-182f-5cbd-7a8569c5232a/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-12T18:14:36,112 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/085c9431-6410-182f-5cbd-7a8569c5232a/nfs.dump.dir in system properties and HBase conf 2024-11-12T18:14:36,112 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/085c9431-6410-182f-5cbd-7a8569c5232a/java.io.tmpdir in system properties and HBase conf 2024-11-12T18:14:36,113 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/085c9431-6410-182f-5cbd-7a8569c5232a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-12T18:14:36,113 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/085c9431-6410-182f-5cbd-7a8569c5232a/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-12T18:14:36,115 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/085c9431-6410-182f-5cbd-7a8569c5232a/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-12T18:14:37,300 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-12T18:14:37,413 INFO [Time-limited test {}] log.Log(170): Logging initialized @3453ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-12T18:14:37,509 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T18:14:37,596 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T18:14:37,625 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T18:14:37,625 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T18:14:37,627 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-12T18:14:37,642 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T18:14:37,645 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@535b1079{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/085c9431-6410-182f-5cbd-7a8569c5232a/hadoop.log.dir/,AVAILABLE} 2024-11-12T18:14:37,647 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1928b27c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T18:14:37,928 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@22b2d85a{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/085c9431-6410-182f-5cbd-7a8569c5232a/java.io.tmpdir/jetty-localhost-43499-hadoop-hdfs-3_4_1-tests_jar-_-any-11000612300614716785/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-12T18:14:37,939 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1b1d377e{HTTP/1.1, (http/1.1)}{localhost:43499} 2024-11-12T18:14:37,939 INFO [Time-limited test {}] server.Server(415): Started @3980ms 2024-11-12T18:14:38,474 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T18:14:38,485 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T18:14:38,490 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T18:14:38,491 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T18:14:38,491 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-12T18:14:38,497 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5e7c54a9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/085c9431-6410-182f-5cbd-7a8569c5232a/hadoop.log.dir/,AVAILABLE} 2024-11-12T18:14:38,498 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@44fb05ac{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T18:14:38,658 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@799718f9{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/085c9431-6410-182f-5cbd-7a8569c5232a/java.io.tmpdir/jetty-localhost-40787-hadoop-hdfs-3_4_1-tests_jar-_-any-2672632392570294988/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T18:14:38,660 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1db949{HTTP/1.1, (http/1.1)}{localhost:40787} 2024-11-12T18:14:38,661 INFO [Time-limited test {}] server.Server(415): Started @4702ms 2024-11-12T18:14:38,739 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-12T18:14:38,928 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T18:14:38,935 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T18:14:38,946 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T18:14:38,946 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T18:14:38,946 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-12T18:14:38,949 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@666ea939{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/085c9431-6410-182f-5cbd-7a8569c5232a/hadoop.log.dir/,AVAILABLE} 2024-11-12T18:14:38,950 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6194c5f8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T18:14:39,114 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@15fa73e4{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/085c9431-6410-182f-5cbd-7a8569c5232a/java.io.tmpdir/jetty-localhost-37577-hadoop-hdfs-3_4_1-tests_jar-_-any-16633649730904544194/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T18:14:39,115 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5569f220{HTTP/1.1, (http/1.1)}{localhost:37577} 2024-11-12T18:14:39,115 INFO [Time-limited test {}] server.Server(415): Started @5156ms 2024-11-12T18:14:39,118 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-12T18:14:39,190 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T18:14:39,204 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T18:14:39,228 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T18:14:39,229 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T18:14:39,229 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-12T18:14:39,230 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@a5627e3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/085c9431-6410-182f-5cbd-7a8569c5232a/hadoop.log.dir/,AVAILABLE} 2024-11-12T18:14:39,231 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@23c8a032{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T18:14:39,401 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6b2a3683{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/085c9431-6410-182f-5cbd-7a8569c5232a/java.io.tmpdir/jetty-localhost-33439-hadoop-hdfs-3_4_1-tests_jar-_-any-125958874449754006/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T18:14:39,403 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@12184f8f{HTTP/1.1, (http/1.1)}{localhost:33439} 2024-11-12T18:14:39,403 INFO [Time-limited test {}] server.Server(415): Started @5444ms 2024-11-12T18:14:39,406 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-12T18:14:39,411 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/085c9431-6410-182f-5cbd-7a8569c5232a/cluster_1655c978-3304-a2e8-86fb-5151dc4c47d4/data/data3/current/BP-497546805-172.17.0.3-1731435276982/current, will proceed with Du for space computation calculation, 2024-11-12T18:14:39,415 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/085c9431-6410-182f-5cbd-7a8569c5232a/cluster_1655c978-3304-a2e8-86fb-5151dc4c47d4/data/data1/current/BP-497546805-172.17.0.3-1731435276982/current, will proceed with Du for space computation calculation, 2024-11-12T18:14:39,418 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/085c9431-6410-182f-5cbd-7a8569c5232a/cluster_1655c978-3304-a2e8-86fb-5151dc4c47d4/data/data2/current/BP-497546805-172.17.0.3-1731435276982/current, will proceed with Du for space computation calculation, 2024-11-12T18:14:39,426 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/085c9431-6410-182f-5cbd-7a8569c5232a/cluster_1655c978-3304-a2e8-86fb-5151dc4c47d4/data/data4/current/BP-497546805-172.17.0.3-1731435276982/current, will proceed with Du for space computation calculation, 2024-11-12T18:14:39,538 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-12T18:14:39,577 WARN [Thread-82 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-12T18:14:39,660 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc10d7e047d05e105 with lease ID 0xba3a9ccbe92c5c02: Processing first storage report for DS-c1ead257-8ceb-4707-bb84-0752011c7478 from datanode DatanodeRegistration(127.0.0.1:35529, datanodeUuid=b04c2015-2cf6-42a6-84c5-e1651d3d3d79, infoPort=33383, infoSecurePort=0, ipcPort=39149, storageInfo=lv=-57;cid=testClusterID;nsid=1452490900;c=1731435276982) 2024-11-12T18:14:39,662 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc10d7e047d05e105 with lease ID 0xba3a9ccbe92c5c02: from storage DS-c1ead257-8ceb-4707-bb84-0752011c7478 node DatanodeRegistration(127.0.0.1:35529, datanodeUuid=b04c2015-2cf6-42a6-84c5-e1651d3d3d79, infoPort=33383, infoSecurePort=0, ipcPort=39149, storageInfo=lv=-57;cid=testClusterID;nsid=1452490900;c=1731435276982), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-12T18:14:39,662 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe4d1a9612d202d72 with lease ID 0xba3a9ccbe92c5c01: Processing first storage report for DS-daada605-434c-4a56-9ec7-9e1ddc6c103c from datanode DatanodeRegistration(127.0.0.1:42977, datanodeUuid=7b47627a-22db-422c-99a3-f861f494d73e, infoPort=36981, infoSecurePort=0, ipcPort=46035, storageInfo=lv=-57;cid=testClusterID;nsid=1452490900;c=1731435276982) 2024-11-12T18:14:39,663 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe4d1a9612d202d72 with lease ID 0xba3a9ccbe92c5c01: from storage DS-daada605-434c-4a56-9ec7-9e1ddc6c103c node DatanodeRegistration(127.0.0.1:42977, datanodeUuid=7b47627a-22db-422c-99a3-f861f494d73e, infoPort=36981, infoSecurePort=0, ipcPort=46035, storageInfo=lv=-57;cid=testClusterID;nsid=1452490900;c=1731435276982), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-12T18:14:39,663 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe4d1a9612d202d72 with lease ID 0xba3a9ccbe92c5c01: Processing first storage report for DS-980ec3a5-5dcc-402b-a5ac-80642d1b664a from datanode DatanodeRegistration(127.0.0.1:42977, datanodeUuid=7b47627a-22db-422c-99a3-f861f494d73e, infoPort=36981, infoSecurePort=0, ipcPort=46035, storageInfo=lv=-57;cid=testClusterID;nsid=1452490900;c=1731435276982) 2024-11-12T18:14:39,663 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe4d1a9612d202d72 with lease ID 0xba3a9ccbe92c5c01: from storage DS-980ec3a5-5dcc-402b-a5ac-80642d1b664a node DatanodeRegistration(127.0.0.1:42977, datanodeUuid=7b47627a-22db-422c-99a3-f861f494d73e, infoPort=36981, infoSecurePort=0, ipcPort=46035, storageInfo=lv=-57;cid=testClusterID;nsid=1452490900;c=1731435276982), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-12T18:14:39,664 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc10d7e047d05e105 with lease ID 0xba3a9ccbe92c5c02: Processing first storage report for DS-3bb16678-4568-456b-9964-2afea9e86c3b from datanode DatanodeRegistration(127.0.0.1:35529, datanodeUuid=b04c2015-2cf6-42a6-84c5-e1651d3d3d79, infoPort=33383, infoSecurePort=0, ipcPort=39149, storageInfo=lv=-57;cid=testClusterID;nsid=1452490900;c=1731435276982) 2024-11-12T18:14:39,664 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc10d7e047d05e105 with lease ID 0xba3a9ccbe92c5c02: from storage DS-3bb16678-4568-456b-9964-2afea9e86c3b node DatanodeRegistration(127.0.0.1:35529, datanodeUuid=b04c2015-2cf6-42a6-84c5-e1651d3d3d79, infoPort=33383, infoSecurePort=0, ipcPort=39149, storageInfo=lv=-57;cid=testClusterID;nsid=1452490900;c=1731435276982), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-12T18:14:39,737 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/085c9431-6410-182f-5cbd-7a8569c5232a/cluster_1655c978-3304-a2e8-86fb-5151dc4c47d4/data/data5/current/BP-497546805-172.17.0.3-1731435276982/current, will proceed with Du for space computation calculation, 2024-11-12T18:14:39,752 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/085c9431-6410-182f-5cbd-7a8569c5232a/cluster_1655c978-3304-a2e8-86fb-5151dc4c47d4/data/data6/current/BP-497546805-172.17.0.3-1731435276982/current, will proceed with Du for space computation calculation, 2024-11-12T18:14:39,809 WARN [Thread-121 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-12T18:14:39,819 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1e6ba44a2fde2d80 with lease ID 0xba3a9ccbe92c5c03: Processing first storage report for DS-12f17fcf-9e85-422d-9538-7fd753218ef4 from datanode DatanodeRegistration(127.0.0.1:46767, datanodeUuid=afe0ec6f-c4c8-4dd7-a2eb-d843f0a850e1, infoPort=35205, infoSecurePort=0, ipcPort=39773, storageInfo=lv=-57;cid=testClusterID;nsid=1452490900;c=1731435276982) 2024-11-12T18:14:39,819 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1e6ba44a2fde2d80 with lease ID 0xba3a9ccbe92c5c03: from storage DS-12f17fcf-9e85-422d-9538-7fd753218ef4 node DatanodeRegistration(127.0.0.1:46767, datanodeUuid=afe0ec6f-c4c8-4dd7-a2eb-d843f0a850e1, infoPort=35205, infoSecurePort=0, ipcPort=39773, storageInfo=lv=-57;cid=testClusterID;nsid=1452490900;c=1731435276982), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-12T18:14:39,819 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1e6ba44a2fde2d80 with lease ID 0xba3a9ccbe92c5c03: Processing first storage report for DS-02ff3a18-1c72-44dc-b444-281ada8c5760 from datanode DatanodeRegistration(127.0.0.1:46767, datanodeUuid=afe0ec6f-c4c8-4dd7-a2eb-d843f0a850e1, infoPort=35205, infoSecurePort=0, ipcPort=39773, storageInfo=lv=-57;cid=testClusterID;nsid=1452490900;c=1731435276982) 2024-11-12T18:14:39,820 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1e6ba44a2fde2d80 with lease ID 0xba3a9ccbe92c5c03: from storage DS-02ff3a18-1c72-44dc-b444-281ada8c5760 node DatanodeRegistration(127.0.0.1:46767, datanodeUuid=afe0ec6f-c4c8-4dd7-a2eb-d843f0a850e1, infoPort=35205, infoSecurePort=0, ipcPort=39773, storageInfo=lv=-57;cid=testClusterID;nsid=1452490900;c=1731435276982), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-12T18:14:40,192 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/085c9431-6410-182f-5cbd-7a8569c5232a 2024-11-12T18:14:40,387 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/085c9431-6410-182f-5cbd-7a8569c5232a/cluster_1655c978-3304-a2e8-86fb-5151dc4c47d4/zookeeper_0, clientPort=49826, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/085c9431-6410-182f-5cbd-7a8569c5232a/cluster_1655c978-3304-a2e8-86fb-5151dc4c47d4/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/085c9431-6410-182f-5cbd-7a8569c5232a/cluster_1655c978-3304-a2e8-86fb-5151dc4c47d4/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-12T18:14:40,403 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49826 2024-11-12T18:14:40,429 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T18:14:40,434 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T18:14:40,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741825_1001 (size=7) 2024-11-12T18:14:40,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741825_1001 (size=7) 2024-11-12T18:14:40,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46767 is added to blk_1073741825_1001 (size=7) 2024-11-12T18:14:41,190 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d with version=8 2024-11-12T18:14:41,191 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/hbase-staging 2024-11-12T18:14:41,306 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-12T18:14:41,605 INFO [Time-limited test {}] client.ConnectionUtils(128): master/9911683f163c:0 server-side Connection retries=45 2024-11-12T18:14:41,618 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T18:14:41,619 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T18:14:41,625 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T18:14:41,625 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T18:14:41,627 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T18:14:41,821 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-12T18:14:41,905 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-12T18:14:41,917 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-12T18:14:41,921 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T18:14:41,959 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 3367 (auto-detected) 2024-11-12T18:14:41,960 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-12T18:14:41,999 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36779 2024-11-12T18:14:42,025 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36779 connecting to ZooKeeper ensemble=127.0.0.1:49826 2024-11-12T18:14:42,061 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:367790x0, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T18:14:42,067 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36779-0x1003533c26e0000 connected 2024-11-12T18:14:42,128 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T18:14:42,134 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T18:14:42,151 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T18:14:42,158 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d, hbase.cluster.distributed=false 2024-11-12T18:14:42,193 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T18:14:42,208 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36779 2024-11-12T18:14:42,212 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36779 2024-11-12T18:14:42,217 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36779 2024-11-12T18:14:42,230 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36779 2024-11-12T18:14:42,232 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36779 2024-11-12T18:14:42,371 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9911683f163c:0 server-side Connection retries=45 2024-11-12T18:14:42,373 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T18:14:42,374 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T18:14:42,374 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T18:14:42,374 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T18:14:42,374 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T18:14:42,378 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-12T18:14:42,381 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T18:14:42,382 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40543 2024-11-12T18:14:42,385 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40543 connecting to ZooKeeper ensemble=127.0.0.1:49826 2024-11-12T18:14:42,386 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T18:14:42,390 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T18:14:42,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:405430x0, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T18:14:42,424 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40543-0x1003533c26e0001 connected 2024-11-12T18:14:42,425 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40543-0x1003533c26e0001, quorum=127.0.0.1:49826, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T18:14:42,430 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-12T18:14:42,449 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-12T18:14:42,452 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40543-0x1003533c26e0001, quorum=127.0.0.1:49826, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-12T18:14:42,460 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40543-0x1003533c26e0001, quorum=127.0.0.1:49826, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T18:14:42,481 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40543 2024-11-12T18:14:42,483 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40543 2024-11-12T18:14:42,486 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40543 2024-11-12T18:14:42,490 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40543 2024-11-12T18:14:42,490 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40543 2024-11-12T18:14:42,512 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9911683f163c:0 server-side Connection retries=45 2024-11-12T18:14:42,513 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T18:14:42,513 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T18:14:42,513 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T18:14:42,514 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T18:14:42,514 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T18:14:42,514 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-12T18:14:42,515 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T18:14:42,516 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:34847 2024-11-12T18:14:42,518 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34847 connecting to ZooKeeper ensemble=127.0.0.1:49826 2024-11-12T18:14:42,520 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T18:14:42,522 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T18:14:42,528 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:348470x0, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T18:14:42,530 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34847-0x1003533c26e0002 connected 2024-11-12T18:14:42,530 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T18:14:42,531 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-12T18:14:42,536 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-12T18:14:42,538 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-12T18:14:42,541 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T18:14:42,544 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34847 2024-11-12T18:14:42,546 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34847 2024-11-12T18:14:42,547 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34847 2024-11-12T18:14:42,551 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34847 2024-11-12T18:14:42,552 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34847 2024-11-12T18:14:42,573 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9911683f163c:0 server-side Connection retries=45 2024-11-12T18:14:42,574 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T18:14:42,574 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T18:14:42,574 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T18:14:42,574 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T18:14:42,575 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T18:14:42,575 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-12T18:14:42,575 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T18:14:42,576 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:45243 2024-11-12T18:14:42,579 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45243 connecting to ZooKeeper ensemble=127.0.0.1:49826 2024-11-12T18:14:42,580 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T18:14:42,584 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T18:14:42,591 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:452430x0, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T18:14:42,592 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:452430x0, quorum=127.0.0.1:49826, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T18:14:42,592 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-12T18:14:42,593 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45243-0x1003533c26e0003 connected 2024-11-12T18:14:42,597 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-12T18:14:42,602 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-12T18:14:42,605 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T18:14:42,606 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45243 2024-11-12T18:14:42,608 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45243 2024-11-12T18:14:42,616 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45243 2024-11-12T18:14:42,620 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45243 2024-11-12T18:14:42,620 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45243 2024-11-12T18:14:42,643 DEBUG [M:0;9911683f163c:36779 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9911683f163c:36779 2024-11-12T18:14:42,645 INFO [master/9911683f163c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/9911683f163c,36779,1731435281367 2024-11-12T18:14:42,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40543-0x1003533c26e0001, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T18:14:42,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T18:14:42,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T18:14:42,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T18:14:42,658 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9911683f163c,36779,1731435281367 2024-11-12T18:14:42,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40543-0x1003533c26e0001, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-12T18:14:42,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-12T18:14:42,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T18:14:42,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T18:14:42,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-12T18:14:42,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T18:14:42,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40543-0x1003533c26e0001, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T18:14:42,690 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-12T18:14:42,695 INFO [master/9911683f163c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9911683f163c,36779,1731435281367 from backup master directory 2024-11-12T18:14:42,698 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9911683f163c,36779,1731435281367 2024-11-12T18:14:42,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T18:14:42,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T18:14:42,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40543-0x1003533c26e0001, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T18:14:42,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T18:14:42,703 WARN [master/9911683f163c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T18:14:42,703 INFO [master/9911683f163c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9911683f163c,36779,1731435281367 2024-11-12T18:14:42,706 INFO [master/9911683f163c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-12T18:14:42,708 INFO [master/9911683f163c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-12T18:14:42,786 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/hbase.id] with ID: 8ba834a2-b172-4942-909b-d8381e92fea0 2024-11-12T18:14:42,787 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/.tmp/hbase.id 2024-11-12T18:14:42,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741826_1002 (size=42) 2024-11-12T18:14:42,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741826_1002 (size=42) 2024-11-12T18:14:42,809 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/.tmp/hbase.id]:[hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/hbase.id] 2024-11-12T18:14:42,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46767 is added to blk_1073741826_1002 (size=42) 2024-11-12T18:14:42,860 INFO [master/9911683f163c:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T18:14:42,866 INFO [master/9911683f163c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-12T18:14:42,894 INFO [master/9911683f163c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 24ms. 2024-11-12T18:14:42,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40543-0x1003533c26e0001, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T18:14:42,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T18:14:42,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T18:14:42,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T18:14:42,913 WARN [IPC Server handler 3 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-12T18:14:42,913 WARN [IPC Server handler 3 on default port 38437 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-12T18:14:42,913 WARN [IPC Server handler 3 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-12T18:14:42,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741827_1003 (size=196) 2024-11-12T18:14:42,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741827_1003 (size=196) 2024-11-12T18:14:42,944 INFO [master/9911683f163c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T18:14:42,947 INFO [master/9911683f163c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-12T18:14:42,965 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:42,970 INFO [master/9911683f163c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T18:14:43,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741828_1004 (size=1189) 2024-11-12T18:14:43,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741828_1004 (size=1189) 2024-11-12T18:14:43,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46767 is added to blk_1073741828_1004 (size=1189) 2024-11-12T18:14:43,029 INFO [master/9911683f163c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/MasterData/data/master/store 2024-11-12T18:14:43,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741829_1005 (size=34) 2024-11-12T18:14:43,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741829_1005 (size=34) 2024-11-12T18:14:43,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46767 is added to blk_1073741829_1005 (size=34) 2024-11-12T18:14:43,057 INFO [master/9911683f163c:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-12T18:14:43,061 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:43,063 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T18:14:43,063 INFO [master/9911683f163c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T18:14:43,063 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T18:14:43,065 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T18:14:43,065 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T18:14:43,066 INFO [master/9911683f163c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T18:14:43,067 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731435283062Disabling compacts and flushes for region at 1731435283062Disabling writes for close at 1731435283065 (+3 ms)Writing region close event to WAL at 1731435283066 (+1 ms)Closed at 1731435283066 2024-11-12T18:14:43,070 WARN [master/9911683f163c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/MasterData/data/master/store/.initializing 2024-11-12T18:14:43,070 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/MasterData/WALs/9911683f163c,36779,1731435281367 2024-11-12T18:14:43,082 INFO [master/9911683f163c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T18:14:43,103 INFO [master/9911683f163c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9911683f163c%2C36779%2C1731435281367, suffix=, logDir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/MasterData/WALs/9911683f163c,36779,1731435281367, archiveDir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/MasterData/oldWALs, maxLogs=10 2024-11-12T18:14:43,128 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/MasterData/WALs/9911683f163c,36779,1731435281367/9911683f163c%2C36779%2C1731435281367.1731435283108, exclude list is [], retry=0 2024-11-12T18:14:43,147 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46767,DS-12f17fcf-9e85-422d-9538-7fd753218ef4,DISK] 2024-11-12T18:14:43,147 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35529,DS-c1ead257-8ceb-4707-bb84-0752011c7478,DISK] 2024-11-12T18:14:43,147 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42977,DS-daada605-434c-4a56-9ec7-9e1ddc6c103c,DISK] 2024-11-12T18:14:43,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-12T18:14:43,196 INFO [master/9911683f163c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/MasterData/WALs/9911683f163c,36779,1731435281367/9911683f163c%2C36779%2C1731435281367.1731435283108 2024-11-12T18:14:43,198 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35205:35205),(127.0.0.1/127.0.0.1:36981:36981),(127.0.0.1/127.0.0.1:33383:33383)] 2024-11-12T18:14:43,198 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-12T18:14:43,199 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:43,202 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T18:14:43,203 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T18:14:43,252 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T18:14:43,281 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-12T18:14:43,285 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:43,288 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T18:14:43,289 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T18:14:43,292 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-12T18:14:43,292 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:43,293 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:43,294 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T18:14:43,296 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-12T18:14:43,296 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:43,297 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:43,297 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T18:14:43,300 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-12T18:14:43,300 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:43,301 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:43,301 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T18:14:43,305 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T18:14:43,306 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T18:14:43,311 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T18:14:43,311 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T18:14:43,315 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T18:14:43,319 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T18:14:43,323 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T18:14:43,325 INFO [master/9911683f163c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64298350, jitterRate=-0.04187992215156555}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T18:14:43,331 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731435283217Initializing all the Stores at 1731435283219 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731435283220 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435283220Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435283221 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435283221Cleaning up temporary data from old regions at 1731435283311 (+90 ms)Region opened successfully at 1731435283331 (+20 ms) 2024-11-12T18:14:43,332 INFO [master/9911683f163c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-12T18:14:43,368 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bc7b0da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9911683f163c/172.17.0.3:0 2024-11-12T18:14:43,403 INFO [master/9911683f163c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-12T18:14:43,415 INFO [master/9911683f163c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T18:14:43,415 INFO [master/9911683f163c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T18:14:43,419 INFO [master/9911683f163c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T18:14:43,420 INFO [master/9911683f163c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-12T18:14:43,425 INFO [master/9911683f163c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-12T18:14:43,426 INFO [master/9911683f163c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-12T18:14:43,453 INFO [master/9911683f163c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-12T18:14:43,463 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-12T18:14:43,465 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-12T18:14:43,468 INFO [master/9911683f163c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-12T18:14:43,469 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-12T18:14:43,471 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-12T18:14:43,473 INFO [master/9911683f163c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-12T18:14:43,477 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-12T18:14:43,479 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-12T18:14:43,480 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-12T18:14:43,482 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-12T18:14:43,502 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-12T18:14:43,504 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-12T18:14:43,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T18:14:43,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40543-0x1003533c26e0001, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T18:14:43,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T18:14:43,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T18:14:43,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T18:14:43,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T18:14:43,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T18:14:43,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40543-0x1003533c26e0001, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T18:14:43,512 INFO [master/9911683f163c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=9911683f163c,36779,1731435281367, sessionid=0x1003533c26e0000, setting cluster-up flag (Was=false) 2024-11-12T18:14:43,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T18:14:43,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T18:14:43,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40543-0x1003533c26e0001, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T18:14:43,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T18:14:43,531 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-12T18:14:43,533 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9911683f163c,36779,1731435281367 2024-11-12T18:14:43,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T18:14:43,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T18:14:43,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40543-0x1003533c26e0001, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T18:14:43,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T18:14:43,545 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-12T18:14:43,547 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9911683f163c,36779,1731435281367 2024-11-12T18:14:43,554 INFO [master/9911683f163c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-12T18:14:43,625 INFO [RS:1;9911683f163c:34847 {}] regionserver.HRegionServer(746): ClusterId : 8ba834a2-b172-4942-909b-d8381e92fea0 2024-11-12T18:14:43,625 INFO [RS:0;9911683f163c:40543 {}] regionserver.HRegionServer(746): ClusterId : 8ba834a2-b172-4942-909b-d8381e92fea0 2024-11-12T18:14:43,625 INFO [RS:2;9911683f163c:45243 {}] regionserver.HRegionServer(746): ClusterId : 8ba834a2-b172-4942-909b-d8381e92fea0 2024-11-12T18:14:43,628 DEBUG [RS:0;9911683f163c:40543 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-12T18:14:43,628 DEBUG [RS:1;9911683f163c:34847 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-12T18:14:43,628 DEBUG [RS:2;9911683f163c:45243 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-12T18:14:43,632 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T18:14:43,634 DEBUG [RS:2;9911683f163c:45243 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-12T18:14:43,634 DEBUG [RS:0;9911683f163c:40543 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-12T18:14:43,634 DEBUG [RS:1;9911683f163c:34847 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-12T18:14:43,634 DEBUG [RS:2;9911683f163c:45243 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-12T18:14:43,634 DEBUG [RS:0;9911683f163c:40543 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-12T18:14:43,634 DEBUG [RS:1;9911683f163c:34847 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-12T18:14:43,638 DEBUG [RS:0;9911683f163c:40543 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-12T18:14:43,638 DEBUG [RS:2;9911683f163c:45243 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-12T18:14:43,638 DEBUG [RS:0;9911683f163c:40543 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e748444, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9911683f163c/172.17.0.3:0 2024-11-12T18:14:43,638 DEBUG [RS:2;9911683f163c:45243 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52418db4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9911683f163c/172.17.0.3:0 2024-11-12T18:14:43,644 DEBUG [RS:1;9911683f163c:34847 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-12T18:14:43,645 DEBUG [RS:1;9911683f163c:34847 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26b0f9c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9911683f163c/172.17.0.3:0 2024-11-12T18:14:43,650 INFO [master/9911683f163c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-12T18:14:43,654 DEBUG [RS:2;9911683f163c:45243 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;9911683f163c:45243 2024-11-12T18:14:43,658 INFO [RS:2;9911683f163c:45243 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-12T18:14:43,658 INFO [RS:2;9911683f163c:45243 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-12T18:14:43,658 DEBUG [RS:2;9911683f163c:45243 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-12T18:14:43,659 INFO [master/9911683f163c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-12T18:14:43,659 DEBUG [RS:0;9911683f163c:40543 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9911683f163c:40543 2024-11-12T18:14:43,660 INFO [RS:0;9911683f163c:40543 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-12T18:14:43,660 INFO [RS:0;9911683f163c:40543 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-12T18:14:43,660 DEBUG [RS:0;9911683f163c:40543 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-12T18:14:43,662 INFO [RS:2;9911683f163c:45243 {}] regionserver.HRegionServer(2659): reportForDuty to master=9911683f163c,36779,1731435281367 with port=45243, startcode=1731435282573 2024-11-12T18:14:43,662 INFO [RS:0;9911683f163c:40543 {}] regionserver.HRegionServer(2659): reportForDuty to master=9911683f163c,36779,1731435281367 with port=40543, startcode=1731435282322 2024-11-12T18:14:43,666 DEBUG [RS:1;9911683f163c:34847 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;9911683f163c:34847 2024-11-12T18:14:43,667 INFO [RS:1;9911683f163c:34847 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-12T18:14:43,667 INFO [RS:1;9911683f163c:34847 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-12T18:14:43,667 DEBUG [RS:1;9911683f163c:34847 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-12T18:14:43,670 INFO [RS:1;9911683f163c:34847 {}] regionserver.HRegionServer(2659): reportForDuty to master=9911683f163c,36779,1731435281367 with port=34847, startcode=1731435282512 2024-11-12T18:14:43,668 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9911683f163c,36779,1731435281367 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-12T18:14:43,677 DEBUG [RS:1;9911683f163c:34847 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-12T18:14:43,677 DEBUG [RS:0;9911683f163c:40543 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-12T18:14:43,677 DEBUG [RS:2;9911683f163c:45243 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-12T18:14:43,677 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9911683f163c:0, corePoolSize=5, maxPoolSize=5 2024-11-12T18:14:43,678 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9911683f163c:0, corePoolSize=5, maxPoolSize=5 2024-11-12T18:14:43,678 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9911683f163c:0, corePoolSize=5, maxPoolSize=5 2024-11-12T18:14:43,678 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9911683f163c:0, corePoolSize=5, maxPoolSize=5 2024-11-12T18:14:43,678 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9911683f163c:0, corePoolSize=10, maxPoolSize=10 2024-11-12T18:14:43,678 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:14:43,679 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9911683f163c:0, corePoolSize=2, maxPoolSize=2 2024-11-12T18:14:43,679 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:14:43,684 INFO [master/9911683f163c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731435313684 2024-11-12T18:14:43,686 INFO [master/9911683f163c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-12T18:14:43,687 INFO [master/9911683f163c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-12T18:14:43,691 INFO [master/9911683f163c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-12T18:14:43,692 INFO [master/9911683f163c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-12T18:14:43,692 INFO [master/9911683f163c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-12T18:14:43,692 INFO [master/9911683f163c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-12T18:14:43,696 INFO [master/9911683f163c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T18:14:43,703 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T18:14:43,704 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-12T18:14:43,711 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:43,712 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-12T18:14:43,713 INFO [master/9911683f163c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-12T18:14:43,715 INFO [master/9911683f163c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-12T18:14:43,715 INFO [master/9911683f163c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-12T18:14:43,718 INFO [master/9911683f163c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-12T18:14:43,719 INFO [master/9911683f163c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-12T18:14:43,719 WARN [IPC Server handler 2 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-12T18:14:43,719 WARN [IPC Server handler 2 on default port 38437 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-12T18:14:43,720 WARN [IPC Server handler 2 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-12T18:14:43,722 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9911683f163c:0:becomeActiveMaster-HFileCleaner.large.0-1731435283720,5,FailOnTimeoutGroup] 2024-11-12T18:14:43,730 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9911683f163c:0:becomeActiveMaster-HFileCleaner.small.0-1731435283723,5,FailOnTimeoutGroup] 2024-11-12T18:14:43,730 INFO [master/9911683f163c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T18:14:43,731 INFO [master/9911683f163c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-12T18:14:43,735 INFO [master/9911683f163c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-12T18:14:43,735 INFO [master/9911683f163c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-12T18:14:43,736 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50591, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-12T18:14:43,736 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42591, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-12T18:14:43,736 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41301, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-12T18:14:43,742 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36779 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9911683f163c,34847,1731435282512 2024-11-12T18:14:43,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741831_1007 (size=1321) 2024-11-12T18:14:43,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741831_1007 (size=1321) 2024-11-12T18:14:43,747 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-12T18:14:43,747 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36779 {}] master.ServerManager(517): Registering regionserver=9911683f163c,34847,1731435282512 2024-11-12T18:14:43,747 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d 2024-11-12T18:14:43,754 WARN [IPC Server handler 0 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-12T18:14:43,754 WARN [IPC Server handler 0 on default port 38437 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-12T18:14:43,754 WARN [IPC Server handler 0 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-12T18:14:43,759 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36779 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9911683f163c,40543,1731435282322 2024-11-12T18:14:43,759 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36779 {}] master.ServerManager(517): Registering regionserver=9911683f163c,40543,1731435282322 2024-11-12T18:14:43,764 DEBUG [RS:1;9911683f163c:34847 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d 2024-11-12T18:14:43,764 DEBUG [RS:1;9911683f163c:34847 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38437 2024-11-12T18:14:43,764 DEBUG [RS:1;9911683f163c:34847 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-12T18:14:43,765 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36779 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9911683f163c,45243,1731435282573 2024-11-12T18:14:43,765 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36779 {}] master.ServerManager(517): Registering regionserver=9911683f163c,45243,1731435282573 2024-11-12T18:14:43,765 DEBUG [RS:0;9911683f163c:40543 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d 2024-11-12T18:14:43,766 DEBUG [RS:0;9911683f163c:40543 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38437 2024-11-12T18:14:43,766 DEBUG [RS:0;9911683f163c:40543 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-12T18:14:43,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741832_1008 (size=32) 2024-11-12T18:14:43,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741832_1008 (size=32) 2024-11-12T18:14:43,771 DEBUG [RS:2;9911683f163c:45243 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d 2024-11-12T18:14:43,771 DEBUG [RS:2;9911683f163c:45243 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38437 2024-11-12T18:14:43,771 DEBUG [RS:2;9911683f163c:45243 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-12T18:14:43,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T18:14:43,773 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:43,774 DEBUG [RS:0;9911683f163c:40543 {}] zookeeper.ZKUtil(111): regionserver:40543-0x1003533c26e0001, quorum=127.0.0.1:49826, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9911683f163c,40543,1731435282322 2024-11-12T18:14:43,774 DEBUG [RS:1;9911683f163c:34847 {}] zookeeper.ZKUtil(111): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9911683f163c,34847,1731435282512 2024-11-12T18:14:43,774 WARN [RS:0;9911683f163c:40543 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T18:14:43,774 WARN [RS:1;9911683f163c:34847 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T18:14:43,774 INFO [RS:0;9911683f163c:40543 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T18:14:43,774 INFO [RS:1;9911683f163c:34847 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T18:14:43,775 DEBUG [RS:1;9911683f163c:34847 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,34847,1731435282512 2024-11-12T18:14:43,775 DEBUG [RS:0;9911683f163c:40543 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,40543,1731435282322 2024-11-12T18:14:43,776 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9911683f163c,40543,1731435282322] 2024-11-12T18:14:43,776 DEBUG [RS:2;9911683f163c:45243 {}] zookeeper.ZKUtil(111): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9911683f163c,45243,1731435282573 2024-11-12T18:14:43,777 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9911683f163c,34847,1731435282512] 2024-11-12T18:14:43,777 WARN [RS:2;9911683f163c:45243 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T18:14:43,777 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9911683f163c,45243,1731435282573] 2024-11-12T18:14:43,777 INFO [RS:2;9911683f163c:45243 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T18:14:43,777 DEBUG [RS:2;9911683f163c:45243 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,45243,1731435282573 2024-11-12T18:14:43,781 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-12T18:14:43,784 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T18:14:43,784 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:43,786 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T18:14:43,786 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-12T18:14:43,789 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T18:14:43,789 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:43,790 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T18:14:43,791 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-12T18:14:43,794 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T18:14:43,794 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:43,795 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T18:14:43,795 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-12T18:14:43,801 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T18:14:43,801 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:43,802 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T18:14:43,803 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T18:14:43,804 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/hbase/meta/1588230740 2024-11-12T18:14:43,805 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/hbase/meta/1588230740 2024-11-12T18:14:43,809 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T18:14:43,809 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T18:14:43,810 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T18:14:43,813 INFO [RS:1;9911683f163c:34847 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-12T18:14:43,813 INFO [RS:2;9911683f163c:45243 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-12T18:14:43,813 INFO [RS:0;9911683f163c:40543 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-12T18:14:43,813 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T18:14:43,818 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T18:14:43,819 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60809404, jitterRate=-0.09386926889419556}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T18:14:43,823 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731435283773Initializing all the Stores at 1731435283776 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731435283776Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731435283781 (+5 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435283781Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731435283781Cleaning up temporary data from old regions at 1731435283809 (+28 ms)Region opened successfully at 1731435283823 (+14 ms) 2024-11-12T18:14:43,823 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T18:14:43,823 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T18:14:43,823 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T18:14:43,823 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T18:14:43,823 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T18:14:43,825 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T18:14:43,826 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731435283823Disabling compacts and flushes for region at 1731435283823Disabling writes for close at 1731435283823Writing region close event to WAL at 1731435283825 (+2 ms)Closed at 1731435283825 2024-11-12T18:14:43,829 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T18:14:43,829 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-12T18:14:43,832 INFO [RS:1;9911683f163c:34847 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-12T18:14:43,832 INFO [RS:2;9911683f163c:45243 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-12T18:14:43,836 INFO [RS:0;9911683f163c:40543 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-12T18:14:43,837 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-12T18:14:43,848 INFO [RS:2;9911683f163c:45243 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T18:14:43,849 INFO [RS:0;9911683f163c:40543 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T18:14:43,849 INFO [RS:2;9911683f163c:45243 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T18:14:43,849 INFO [RS:1;9911683f163c:34847 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T18:14:43,849 INFO [RS:0;9911683f163c:40543 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T18:14:43,849 INFO [RS:1;9911683f163c:34847 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T18:14:43,853 INFO [RS:2;9911683f163c:45243 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-12T18:14:43,854 INFO [RS:0;9911683f163c:40543 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-12T18:14:43,857 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T18:14:43,861 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-12T18:14:43,863 INFO [RS:0;9911683f163c:40543 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-12T18:14:43,863 INFO [RS:2;9911683f163c:45243 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-12T18:14:43,863 INFO [RS:1;9911683f163c:34847 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-12T18:14:43,864 INFO [RS:1;9911683f163c:34847 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-12T18:14:43,865 INFO [RS:2;9911683f163c:45243 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-12T18:14:43,865 INFO [RS:1;9911683f163c:34847 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-12T18:14:43,865 INFO [RS:0;9911683f163c:40543 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-12T18:14:43,865 DEBUG [RS:2;9911683f163c:45243 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:14:43,865 DEBUG [RS:1;9911683f163c:34847 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:14:43,865 DEBUG [RS:0;9911683f163c:40543 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:14:43,865 DEBUG [RS:2;9911683f163c:45243 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:14:43,865 DEBUG [RS:1;9911683f163c:34847 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:14:43,865 DEBUG [RS:0;9911683f163c:40543 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:14:43,866 DEBUG [RS:0;9911683f163c:40543 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:14:43,866 DEBUG [RS:2;9911683f163c:45243 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:14:43,866 DEBUG [RS:1;9911683f163c:34847 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:14:43,866 DEBUG [RS:0;9911683f163c:40543 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:14:43,866 DEBUG [RS:2;9911683f163c:45243 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:14:43,866 DEBUG [RS:1;9911683f163c:34847 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:14:43,866 DEBUG [RS:0;9911683f163c:40543 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:14:43,866 DEBUG [RS:2;9911683f163c:45243 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:14:43,866 DEBUG [RS:0;9911683f163c:40543 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9911683f163c:0, corePoolSize=2, maxPoolSize=2 2024-11-12T18:14:43,866 DEBUG [RS:1;9911683f163c:34847 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:14:43,866 DEBUG [RS:0;9911683f163c:40543 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:14:43,866 DEBUG [RS:2;9911683f163c:45243 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9911683f163c:0, corePoolSize=2, maxPoolSize=2 2024-11-12T18:14:43,866 DEBUG [RS:0;9911683f163c:40543 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:14:43,866 DEBUG [RS:1;9911683f163c:34847 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9911683f163c:0, corePoolSize=2, maxPoolSize=2 2024-11-12T18:14:43,866 DEBUG [RS:2;9911683f163c:45243 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:14:43,866 DEBUG [RS:0;9911683f163c:40543 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:14:43,866 DEBUG [RS:1;9911683f163c:34847 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:14:43,867 DEBUG [RS:0;9911683f163c:40543 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:14:43,867 DEBUG [RS:2;9911683f163c:45243 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:14:43,867 DEBUG [RS:1;9911683f163c:34847 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:14:43,867 DEBUG [RS:0;9911683f163c:40543 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:14:43,867 DEBUG [RS:2;9911683f163c:45243 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:14:43,867 DEBUG [RS:0;9911683f163c:40543 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:14:43,867 DEBUG [RS:1;9911683f163c:34847 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:14:43,867 DEBUG [RS:0;9911683f163c:40543 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9911683f163c:0, corePoolSize=3, maxPoolSize=3 2024-11-12T18:14:43,867 DEBUG [RS:2;9911683f163c:45243 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:14:43,867 DEBUG [RS:1;9911683f163c:34847 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:14:43,867 DEBUG [RS:0;9911683f163c:40543 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0, corePoolSize=3, maxPoolSize=3 2024-11-12T18:14:43,867 DEBUG [RS:2;9911683f163c:45243 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:14:43,867 DEBUG [RS:1;9911683f163c:34847 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:14:43,867 DEBUG [RS:2;9911683f163c:45243 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:14:43,867 DEBUG [RS:1;9911683f163c:34847 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:14:43,867 DEBUG [RS:1;9911683f163c:34847 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9911683f163c:0, corePoolSize=3, maxPoolSize=3 2024-11-12T18:14:43,867 DEBUG [RS:2;9911683f163c:45243 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9911683f163c:0, corePoolSize=3, maxPoolSize=3 2024-11-12T18:14:43,868 DEBUG [RS:1;9911683f163c:34847 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0, corePoolSize=3, maxPoolSize=3 2024-11-12T18:14:43,868 DEBUG [RS:2;9911683f163c:45243 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0, corePoolSize=3, maxPoolSize=3 2024-11-12T18:14:43,876 INFO [RS:0;9911683f163c:40543 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T18:14:43,877 INFO [RS:0;9911683f163c:40543 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T18:14:43,877 INFO [RS:0;9911683f163c:40543 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T18:14:43,877 INFO [RS:1;9911683f163c:34847 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T18:14:43,877 INFO [RS:0;9911683f163c:40543 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-12T18:14:43,877 INFO [RS:1;9911683f163c:34847 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T18:14:43,877 INFO [RS:0;9911683f163c:40543 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-12T18:14:43,877 INFO [RS:0;9911683f163c:40543 {}] hbase.ChoreService(168): Chore ScheduledChore name=9911683f163c,40543,1731435282322-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T18:14:43,877 INFO [RS:1;9911683f163c:34847 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T18:14:43,877 INFO [RS:1;9911683f163c:34847 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-12T18:14:43,877 INFO [RS:1;9911683f163c:34847 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-12T18:14:43,877 INFO [RS:1;9911683f163c:34847 {}] hbase.ChoreService(168): Chore ScheduledChore name=9911683f163c,34847,1731435282512-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T18:14:43,880 INFO [RS:2;9911683f163c:45243 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T18:14:43,881 INFO [RS:2;9911683f163c:45243 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T18:14:43,881 INFO [RS:2;9911683f163c:45243 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T18:14:43,881 INFO [RS:2;9911683f163c:45243 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-12T18:14:43,881 INFO [RS:2;9911683f163c:45243 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-12T18:14:43,881 INFO [RS:2;9911683f163c:45243 {}] hbase.ChoreService(168): Chore ScheduledChore name=9911683f163c,45243,1731435282573-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T18:14:43,904 INFO [RS:2;9911683f163c:45243 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-12T18:14:43,904 INFO [RS:1;9911683f163c:34847 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-12T18:14:43,904 INFO [RS:0;9911683f163c:40543 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-12T18:14:43,907 INFO [RS:1;9911683f163c:34847 {}] hbase.ChoreService(168): Chore ScheduledChore name=9911683f163c,34847,1731435282512-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T18:14:43,907 INFO [RS:2;9911683f163c:45243 {}] hbase.ChoreService(168): Chore ScheduledChore name=9911683f163c,45243,1731435282573-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T18:14:43,907 INFO [RS:0;9911683f163c:40543 {}] hbase.ChoreService(168): Chore ScheduledChore name=9911683f163c,40543,1731435282322-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T18:14:43,907 INFO [RS:1;9911683f163c:34847 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T18:14:43,907 INFO [RS:2;9911683f163c:45243 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T18:14:43,907 INFO [RS:0;9911683f163c:40543 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T18:14:43,908 INFO [RS:1;9911683f163c:34847 {}] regionserver.Replication(171): 9911683f163c,34847,1731435282512 started 2024-11-12T18:14:43,908 INFO [RS:2;9911683f163c:45243 {}] regionserver.Replication(171): 9911683f163c,45243,1731435282573 started 2024-11-12T18:14:43,908 INFO [RS:0;9911683f163c:40543 {}] regionserver.Replication(171): 9911683f163c,40543,1731435282322 started 2024-11-12T18:14:43,939 INFO [RS:0;9911683f163c:40543 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T18:14:43,939 INFO [RS:1;9911683f163c:34847 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T18:14:43,939 INFO [RS:2;9911683f163c:45243 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T18:14:43,939 INFO [RS:2;9911683f163c:45243 {}] regionserver.HRegionServer(1482): Serving as 9911683f163c,45243,1731435282573, RpcServer on 9911683f163c/172.17.0.3:45243, sessionid=0x1003533c26e0003 2024-11-12T18:14:43,939 INFO [RS:1;9911683f163c:34847 {}] regionserver.HRegionServer(1482): Serving as 9911683f163c,34847,1731435282512, RpcServer on 9911683f163c/172.17.0.3:34847, sessionid=0x1003533c26e0002 2024-11-12T18:14:43,939 INFO [RS:0;9911683f163c:40543 {}] regionserver.HRegionServer(1482): Serving as 9911683f163c,40543,1731435282322, RpcServer on 9911683f163c/172.17.0.3:40543, sessionid=0x1003533c26e0001 2024-11-12T18:14:43,940 DEBUG [RS:0;9911683f163c:40543 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-12T18:14:43,940 DEBUG [RS:2;9911683f163c:45243 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-12T18:14:43,940 DEBUG [RS:1;9911683f163c:34847 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-12T18:14:43,940 DEBUG [RS:0;9911683f163c:40543 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9911683f163c,40543,1731435282322 2024-11-12T18:14:43,940 DEBUG [RS:2;9911683f163c:45243 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9911683f163c,45243,1731435282573 2024-11-12T18:14:43,940 DEBUG [RS:1;9911683f163c:34847 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9911683f163c,34847,1731435282512 2024-11-12T18:14:43,941 DEBUG [RS:0;9911683f163c:40543 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9911683f163c,40543,1731435282322' 2024-11-12T18:14:43,941 DEBUG [RS:2;9911683f163c:45243 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9911683f163c,45243,1731435282573' 2024-11-12T18:14:43,941 DEBUG [RS:1;9911683f163c:34847 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9911683f163c,34847,1731435282512' 2024-11-12T18:14:43,941 DEBUG [RS:0;9911683f163c:40543 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-12T18:14:43,941 DEBUG [RS:2;9911683f163c:45243 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-12T18:14:43,941 DEBUG [RS:1;9911683f163c:34847 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-12T18:14:43,942 DEBUG [RS:2;9911683f163c:45243 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-12T18:14:43,942 DEBUG [RS:0;9911683f163c:40543 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-12T18:14:43,942 DEBUG [RS:1;9911683f163c:34847 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-12T18:14:43,942 DEBUG [RS:0;9911683f163c:40543 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-12T18:14:43,942 DEBUG [RS:1;9911683f163c:34847 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-12T18:14:43,942 DEBUG [RS:2;9911683f163c:45243 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-12T18:14:43,942 DEBUG [RS:0;9911683f163c:40543 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-12T18:14:43,943 DEBUG [RS:2;9911683f163c:45243 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-12T18:14:43,943 DEBUG [RS:1;9911683f163c:34847 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-12T18:14:43,943 DEBUG [RS:0;9911683f163c:40543 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9911683f163c,40543,1731435282322 2024-11-12T18:14:43,943 DEBUG [RS:2;9911683f163c:45243 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9911683f163c,45243,1731435282573 2024-11-12T18:14:43,943 DEBUG [RS:0;9911683f163c:40543 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9911683f163c,40543,1731435282322' 2024-11-12T18:14:43,943 DEBUG [RS:1;9911683f163c:34847 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9911683f163c,34847,1731435282512 2024-11-12T18:14:43,943 DEBUG [RS:2;9911683f163c:45243 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9911683f163c,45243,1731435282573' 2024-11-12T18:14:43,943 DEBUG [RS:0;9911683f163c:40543 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-12T18:14:43,943 DEBUG [RS:1;9911683f163c:34847 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9911683f163c,34847,1731435282512' 2024-11-12T18:14:43,943 DEBUG [RS:2;9911683f163c:45243 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-12T18:14:43,943 DEBUG [RS:1;9911683f163c:34847 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-12T18:14:43,944 DEBUG [RS:0;9911683f163c:40543 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-12T18:14:43,944 DEBUG [RS:2;9911683f163c:45243 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-12T18:14:43,944 DEBUG [RS:1;9911683f163c:34847 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-12T18:14:43,944 DEBUG [RS:0;9911683f163c:40543 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-12T18:14:43,944 INFO [RS:0;9911683f163c:40543 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-12T18:14:43,944 DEBUG [RS:2;9911683f163c:45243 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-12T18:14:43,944 INFO [RS:0;9911683f163c:40543 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-12T18:14:43,944 INFO [RS:2;9911683f163c:45243 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-12T18:14:43,945 INFO [RS:2;9911683f163c:45243 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-12T18:14:43,945 DEBUG [RS:1;9911683f163c:34847 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-12T18:14:43,945 INFO [RS:1;9911683f163c:34847 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-12T18:14:43,945 INFO [RS:1;9911683f163c:34847 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-12T18:14:44,012 WARN [9911683f163c:36779 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-12T18:14:44,050 INFO [RS:1;9911683f163c:34847 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T18:14:44,050 INFO [RS:2;9911683f163c:45243 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T18:14:44,050 INFO [RS:0;9911683f163c:40543 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T18:14:44,054 INFO [RS:1;9911683f163c:34847 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9911683f163c%2C34847%2C1731435282512, suffix=, logDir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,34847,1731435282512, archiveDir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/oldWALs, maxLogs=32 2024-11-12T18:14:44,054 INFO [RS:0;9911683f163c:40543 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9911683f163c%2C40543%2C1731435282322, suffix=, logDir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,40543,1731435282322, archiveDir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/oldWALs, maxLogs=32 2024-11-12T18:14:44,054 INFO [RS:2;9911683f163c:45243 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9911683f163c%2C45243%2C1731435282573, suffix=, logDir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,45243,1731435282573, archiveDir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/oldWALs, maxLogs=32 2024-11-12T18:14:44,076 DEBUG [RS:2;9911683f163c:45243 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,45243,1731435282573/9911683f163c%2C45243%2C1731435282573.1731435284059, exclude list is [], retry=0 2024-11-12T18:14:44,081 DEBUG [RS:0;9911683f163c:40543 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,40543,1731435282322/9911683f163c%2C40543%2C1731435282322.1731435284060, exclude list is [], retry=0 2024-11-12T18:14:44,081 DEBUG [RS:1;9911683f163c:34847 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,34847,1731435282512/9911683f163c%2C34847%2C1731435282512.1731435284060, exclude list is [], retry=0 2024-11-12T18:14:44,082 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46767,DS-12f17fcf-9e85-422d-9538-7fd753218ef4,DISK] 2024-11-12T18:14:44,082 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42977,DS-daada605-434c-4a56-9ec7-9e1ddc6c103c,DISK] 2024-11-12T18:14:44,082 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35529,DS-c1ead257-8ceb-4707-bb84-0752011c7478,DISK] 2024-11-12T18:14:44,084 WARN [IPC Server handler 2 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-12T18:14:44,084 WARN [IPC Server handler 2 on default port 38437 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-12T18:14:44,084 WARN [IPC Server handler 2 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-12T18:14:44,084 WARN [IPC Server handler 1 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-12T18:14:44,084 WARN [IPC Server handler 1 on default port 38437 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-12T18:14:44,085 WARN [IPC Server handler 1 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-12T18:14:44,126 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35529,DS-c1ead257-8ceb-4707-bb84-0752011c7478,DISK] 2024-11-12T18:14:44,126 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35529,DS-c1ead257-8ceb-4707-bb84-0752011c7478,DISK] 2024-11-12T18:14:44,126 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42977,DS-daada605-434c-4a56-9ec7-9e1ddc6c103c,DISK] 2024-11-12T18:14:44,127 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42977,DS-daada605-434c-4a56-9ec7-9e1ddc6c103c,DISK] 2024-11-12T18:14:44,130 INFO [RS:2;9911683f163c:45243 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,45243,1731435282573/9911683f163c%2C45243%2C1731435282573.1731435284059 2024-11-12T18:14:44,131 DEBUG [RS:2;9911683f163c:45243 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35205:35205),(127.0.0.1/127.0.0.1:33383:33383),(127.0.0.1/127.0.0.1:36981:36981)] 2024-11-12T18:14:44,133 INFO [RS:0;9911683f163c:40543 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,40543,1731435282322/9911683f163c%2C40543%2C1731435282322.1731435284060 2024-11-12T18:14:44,134 DEBUG [RS:0;9911683f163c:40543 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33383:33383),(127.0.0.1/127.0.0.1:36981:36981)] 2024-11-12T18:14:44,134 INFO [RS:1;9911683f163c:34847 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,34847,1731435282512/9911683f163c%2C34847%2C1731435282512.1731435284060 2024-11-12T18:14:44,134 DEBUG [RS:1;9911683f163c:34847 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33383:33383),(127.0.0.1/127.0.0.1:36981:36981)] 2024-11-12T18:14:44,265 DEBUG [9911683f163c:36779 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-12T18:14:44,274 DEBUG [9911683f163c:36779 {}] balancer.BalancerClusterState(204): Hosts are {9911683f163c=0} racks are {/default-rack=0} 2024-11-12T18:14:44,283 DEBUG [9911683f163c:36779 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-12T18:14:44,283 DEBUG [9911683f163c:36779 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-12T18:14:44,283 DEBUG [9911683f163c:36779 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-12T18:14:44,283 DEBUG [9911683f163c:36779 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-12T18:14:44,283 DEBUG [9911683f163c:36779 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-12T18:14:44,283 DEBUG [9911683f163c:36779 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-12T18:14:44,283 INFO [9911683f163c:36779 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-12T18:14:44,283 INFO [9911683f163c:36779 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-12T18:14:44,283 INFO [9911683f163c:36779 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-12T18:14:44,283 DEBUG [9911683f163c:36779 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-12T18:14:44,292 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9911683f163c,40543,1731435282322 2024-11-12T18:14:44,301 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9911683f163c,40543,1731435282322, state=OPENING 2024-11-12T18:14:44,309 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-12T18:14:44,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40543-0x1003533c26e0001, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T18:14:44,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T18:14:44,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T18:14:44,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T18:14:44,312 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:44,312 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:44,313 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:44,313 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:44,315 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T18:14:44,318 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,40543,1731435282322}] 2024-11-12T18:14:44,322 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-12T18:14:44,322 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-12T18:14:44,322 WARN [RedundancyMonitor {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK, ARCHIVE], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-12T18:14:44,322 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) All required storage types are unavailable: unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-12T18:14:44,323 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-12T18:14:44,323 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-12T18:14:44,323 WARN [RedundancyMonitor {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK, ARCHIVE], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-12T18:14:44,323 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) All required storage types are unavailable: unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-12T18:14:44,323 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-12T18:14:44,323 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-12T18:14:44,323 WARN [RedundancyMonitor {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK, ARCHIVE], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-12T18:14:44,323 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) All required storage types are unavailable: unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-12T18:14:44,499 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T18:14:44,502 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36263, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T18:14:44,516 INFO [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-12T18:14:44,516 INFO [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T18:14:44,517 INFO [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-12T18:14:44,520 INFO [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9911683f163c%2C40543%2C1731435282322.meta, suffix=.meta, logDir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,40543,1731435282322, archiveDir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/oldWALs, maxLogs=32 2024-11-12T18:14:44,537 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,40543,1731435282322/9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, exclude list is [], retry=0 2024-11-12T18:14:44,540 WARN [IPC Server handler 2 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-12T18:14:44,540 WARN [IPC Server handler 2 on default port 38437 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-12T18:14:44,540 WARN [IPC Server handler 2 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-12T18:14:44,542 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35529,DS-c1ead257-8ceb-4707-bb84-0752011c7478,DISK] 2024-11-12T18:14:44,542 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42977,DS-daada605-434c-4a56-9ec7-9e1ddc6c103c,DISK] 2024-11-12T18:14:44,545 INFO [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,40543,1731435282322/9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta 2024-11-12T18:14:44,546 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33383:33383),(127.0.0.1/127.0.0.1:36981:36981)] 2024-11-12T18:14:44,546 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-12T18:14:44,548 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-12T18:14:44,551 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-12T18:14:44,556 INFO [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-12T18:14:44,561 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-12T18:14:44,562 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:44,562 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-12T18:14:44,562 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-12T18:14:44,566 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-12T18:14:44,567 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T18:14:44,567 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:44,568 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T18:14:44,568 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-12T18:14:44,570 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T18:14:44,570 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:44,571 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T18:14:44,571 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-12T18:14:44,573 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T18:14:44,573 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:44,573 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T18:14:44,574 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-12T18:14:44,575 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T18:14:44,575 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:44,576 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T18:14:44,576 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T18:14:44,577 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/hbase/meta/1588230740 2024-11-12T18:14:44,579 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/hbase/meta/1588230740 2024-11-12T18:14:44,582 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T18:14:44,582 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T18:14:44,583 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T18:14:44,585 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T18:14:44,587 INFO [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64759244, jitterRate=-0.03501206636428833}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T18:14:44,587 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-12T18:14:44,589 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731435284563Writing region info on filesystem at 1731435284563Initializing all the Stores at 1731435284565 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731435284565Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731435284565Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435284565Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731435284565Cleaning up temporary data from old regions at 1731435284582 (+17 ms)Running coprocessor post-open hooks at 1731435284587 (+5 ms)Region opened successfully at 1731435284589 (+2 ms) 2024-11-12T18:14:44,597 INFO [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731435284488 2024-11-12T18:14:44,610 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-12T18:14:44,610 INFO [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-12T18:14:44,612 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9911683f163c,40543,1731435282322 2024-11-12T18:14:44,614 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9911683f163c,40543,1731435282322, state=OPEN 2024-11-12T18:14:44,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40543-0x1003533c26e0001, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:44,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:44,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:44,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:44,617 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:44,617 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:44,617 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:44,617 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:44,618 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9911683f163c,40543,1731435282322 2024-11-12T18:14:44,623 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-12T18:14:44,623 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,40543,1731435282322 in 300 msec 2024-11-12T18:14:44,631 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-12T18:14:44,631 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 788 msec 2024-11-12T18:14:44,633 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T18:14:44,633 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-12T18:14:44,654 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:14:44,655 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,40543,1731435282322, seqNum=-1] 2024-11-12T18:14:44,679 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:14:44,681 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57361, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:14:44,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1170 sec 2024-11-12T18:14:44,702 INFO [master/9911683f163c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731435284702, completionTime=-1 2024-11-12T18:14:44,705 INFO [master/9911683f163c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-12T18:14:44,705 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-12T18:14:44,736 INFO [master/9911683f163c:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=3 2024-11-12T18:14:44,736 INFO [master/9911683f163c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731435344736 2024-11-12T18:14:44,737 INFO [master/9911683f163c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731435404736 2024-11-12T18:14:44,737 INFO [master/9911683f163c:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 31 msec 2024-11-12T18:14:44,738 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-12T18:14:44,746 INFO [master/9911683f163c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9911683f163c,36779,1731435281367-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T18:14:44,746 INFO [master/9911683f163c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9911683f163c,36779,1731435281367-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T18:14:44,746 INFO [master/9911683f163c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9911683f163c,36779,1731435281367-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T18:14:44,748 INFO [master/9911683f163c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9911683f163c:36779, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T18:14:44,748 INFO [master/9911683f163c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-12T18:14:44,749 INFO [master/9911683f163c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-12T18:14:44,755 DEBUG [master/9911683f163c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-12T18:14:44,779 INFO [master/9911683f163c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.075sec 2024-11-12T18:14:44,781 INFO [master/9911683f163c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-12T18:14:44,783 INFO [master/9911683f163c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-12T18:14:44,784 INFO [master/9911683f163c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-12T18:14:44,785 INFO [master/9911683f163c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-12T18:14:44,785 INFO [master/9911683f163c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-12T18:14:44,786 INFO [master/9911683f163c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9911683f163c,36779,1731435281367-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T18:14:44,787 INFO [master/9911683f163c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9911683f163c,36779,1731435281367-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-12T18:14:44,792 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-12T18:14:44,793 INFO [master/9911683f163c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-12T18:14:44,794 INFO [master/9911683f163c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9911683f163c,36779,1731435281367-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T18:14:44,837 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3665ca09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:44,842 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-12T18:14:44,842 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-12T18:14:44,846 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,36779,-1 for getting cluster id 2024-11-12T18:14:44,849 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:14:44,858 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ba834a2-b172-4942-909b-d8381e92fea0' 2024-11-12T18:14:44,861 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:14:44,861 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ba834a2-b172-4942-909b-d8381e92fea0" 2024-11-12T18:14:44,863 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a5c683d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:44,864 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,36779,-1] 2024-11-12T18:14:44,866 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:14:44,869 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:44,869 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:14:44,872 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43d94417, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:44,873 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:14:44,880 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,40543,1731435282322, seqNum=-1] 2024-11-12T18:14:44,881 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:14:44,884 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39856, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:14:44,909 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=9911683f163c,36779,1731435281367 2024-11-12T18:14:44,913 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T18:14:44,918 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 9911683f163c,36779,1731435281367 2024-11-12T18:14:44,922 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@157f800b 2024-11-12T18:14:44,924 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T18:14:44,927 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53684, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T18:14:44,936 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T18:14:44,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-11-12T18:14:44,974 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T18:14:44,977 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-11-12T18:14:44,978 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:44,981 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T18:14:44,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T18:14:44,989 WARN [IPC Server handler 4 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-12T18:14:44,989 WARN [IPC Server handler 4 on default port 38437 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-12T18:14:44,990 WARN [IPC Server handler 4 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-12T18:14:45,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741837_1013 (size=358) 2024-11-12T18:14:45,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741837_1013 (size=358) 2024-11-12T18:14:45,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T18:14:45,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T18:14:45,409 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 3a62a194de5221e9f84b2cfd2e83a46a, NAME => 'test,ccc,1731435284929.3a62a194de5221e9f84b2cfd2e83a46a.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d 2024-11-12T18:14:45,409 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5dcfe82d27e00f564e4ad71ea2e78f5c, NAME => 'test,,1731435284929.5dcfe82d27e00f564e4ad71ea2e78f5c.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d 2024-11-12T18:14:45,409 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 2248a0d1296d1dbe4f8c9805798db03c, NAME => 'test,bbb,1731435284929.2248a0d1296d1dbe4f8c9805798db03c.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d 2024-11-12T18:14:45,414 WARN [IPC Server handler 2 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-12T18:14:45,414 WARN [IPC Server handler 4 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-12T18:14:45,414 WARN [IPC Server handler 2 on default port 38437 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-12T18:14:45,414 WARN [IPC Server handler 4 on default port 38437 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-12T18:14:45,414 WARN [IPC Server handler 2 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-12T18:14:45,414 WARN [IPC Server handler 4 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-12T18:14:45,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741838_1014 (size=45) 2024-11-12T18:14:45,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741838_1014 (size=45) 2024-11-12T18:14:45,428 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1731435284929.2248a0d1296d1dbe4f8c9805798db03c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:45,428 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 2248a0d1296d1dbe4f8c9805798db03c, disabling compactions & flushes 2024-11-12T18:14:45,428 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1731435284929.2248a0d1296d1dbe4f8c9805798db03c. 2024-11-12T18:14:45,428 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1731435284929.2248a0d1296d1dbe4f8c9805798db03c. 2024-11-12T18:14:45,428 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1731435284929.2248a0d1296d1dbe4f8c9805798db03c. after waiting 0 ms 2024-11-12T18:14:45,429 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1731435284929.2248a0d1296d1dbe4f8c9805798db03c. 2024-11-12T18:14:45,429 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1731435284929.2248a0d1296d1dbe4f8c9805798db03c. 2024-11-12T18:14:45,429 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 2248a0d1296d1dbe4f8c9805798db03c: Waiting for close lock at 1731435285428Disabling compacts and flushes for region at 1731435285428Disabling writes for close at 1731435285429 (+1 ms)Writing region close event to WAL at 1731435285429Closed at 1731435285429 2024-11-12T18:14:45,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741839_1015 (size=45) 2024-11-12T18:14:45,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741839_1015 (size=45) 2024-11-12T18:14:45,430 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 6e290ad15dec09a0cb11782988af7c4c, NAME => 'test,ddd,1731435284929.6e290ad15dec09a0cb11782988af7c4c.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d 2024-11-12T18:14:45,431 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1731435284929.3a62a194de5221e9f84b2cfd2e83a46a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:45,431 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 3a62a194de5221e9f84b2cfd2e83a46a, disabling compactions & flushes 2024-11-12T18:14:45,431 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1731435284929.3a62a194de5221e9f84b2cfd2e83a46a. 2024-11-12T18:14:45,431 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1731435284929.3a62a194de5221e9f84b2cfd2e83a46a. 2024-11-12T18:14:45,431 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1731435284929.3a62a194de5221e9f84b2cfd2e83a46a. after waiting 0 ms 2024-11-12T18:14:45,431 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1731435284929.3a62a194de5221e9f84b2cfd2e83a46a. 2024-11-12T18:14:45,432 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1731435284929.3a62a194de5221e9f84b2cfd2e83a46a. 2024-11-12T18:14:45,432 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 3a62a194de5221e9f84b2cfd2e83a46a: Waiting for close lock at 1731435285431Disabling compacts and flushes for region at 1731435285431Disabling writes for close at 1731435285431Writing region close event to WAL at 1731435285432 (+1 ms)Closed at 1731435285432 2024-11-12T18:14:45,433 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => c0b5ce997df05cef2b0f3fd25e49fafc, NAME => 'test,eee,1731435284929.c0b5ce997df05cef2b0f3fd25e49fafc.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d 2024-11-12T18:14:45,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741840_1016 (size=42) 2024-11-12T18:14:45,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741840_1016 (size=42) 2024-11-12T18:14:45,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46767 is added to blk_1073741840_1016 (size=42) 2024-11-12T18:14:45,437 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1731435284929.5dcfe82d27e00f564e4ad71ea2e78f5c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:45,437 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 5dcfe82d27e00f564e4ad71ea2e78f5c, disabling compactions & flushes 2024-11-12T18:14:45,438 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1731435284929.5dcfe82d27e00f564e4ad71ea2e78f5c. 2024-11-12T18:14:45,438 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1731435284929.5dcfe82d27e00f564e4ad71ea2e78f5c. 2024-11-12T18:14:45,438 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1731435284929.5dcfe82d27e00f564e4ad71ea2e78f5c. after waiting 0 ms 2024-11-12T18:14:45,438 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1731435284929.5dcfe82d27e00f564e4ad71ea2e78f5c. 2024-11-12T18:14:45,438 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1731435284929.5dcfe82d27e00f564e4ad71ea2e78f5c. 2024-11-12T18:14:45,438 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5dcfe82d27e00f564e4ad71ea2e78f5c: Waiting for close lock at 1731435285437Disabling compacts and flushes for region at 1731435285437Disabling writes for close at 1731435285438 (+1 ms)Writing region close event to WAL at 1731435285438Closed at 1731435285438 2024-11-12T18:14:45,438 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4aff96e33c9923b5d2448f72171fd310, NAME => 'test,fff,1731435284929.4aff96e33c9923b5d2448f72171fd310.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d 2024-11-12T18:14:45,445 WARN [IPC Server handler 2 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-12T18:14:45,445 WARN [IPC Server handler 2 on default port 38437 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-12T18:14:45,445 WARN [IPC Server handler 3 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-12T18:14:45,445 WARN [IPC Server handler 2 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-12T18:14:45,445 WARN [IPC Server handler 0 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-12T18:14:45,446 WARN [IPC Server handler 3 on default port 38437 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-12T18:14:45,446 WARN [IPC Server handler 3 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-12T18:14:45,446 WARN [IPC Server handler 0 on default port 38437 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-12T18:14:45,446 WARN [IPC Server handler 0 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-12T18:14:45,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741841_1017 (size=45) 2024-11-12T18:14:45,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741841_1017 (size=45) 2024-11-12T18:14:45,458 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ddd,1731435284929.6e290ad15dec09a0cb11782988af7c4c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:45,458 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 6e290ad15dec09a0cb11782988af7c4c, disabling compactions & flushes 2024-11-12T18:14:45,458 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ddd,1731435284929.6e290ad15dec09a0cb11782988af7c4c. 2024-11-12T18:14:45,458 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1731435284929.6e290ad15dec09a0cb11782988af7c4c. 2024-11-12T18:14:45,458 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1731435284929.6e290ad15dec09a0cb11782988af7c4c. after waiting 0 ms 2024-11-12T18:14:45,458 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1731435284929.6e290ad15dec09a0cb11782988af7c4c. 2024-11-12T18:14:45,458 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ddd,1731435284929.6e290ad15dec09a0cb11782988af7c4c. 2024-11-12T18:14:45,458 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 6e290ad15dec09a0cb11782988af7c4c: Waiting for close lock at 1731435285458Disabling compacts and flushes for region at 1731435285458Disabling writes for close at 1731435285458Writing region close event to WAL at 1731435285458Closed at 1731435285458 2024-11-12T18:14:45,459 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 26fc65f8daf71d5831131b1bc6746a63, NAME => 'test,ggg,1731435284929.26fc65f8daf71d5831131b1bc6746a63.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d 2024-11-12T18:14:45,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741842_1018 (size=45) 2024-11-12T18:14:45,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741842_1018 (size=45) 2024-11-12T18:14:45,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741843_1019 (size=45) 2024-11-12T18:14:45,470 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,fff,1731435284929.4aff96e33c9923b5d2448f72171fd310.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:45,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741843_1019 (size=45) 2024-11-12T18:14:45,470 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 4aff96e33c9923b5d2448f72171fd310, disabling compactions & flushes 2024-11-12T18:14:45,470 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,fff,1731435284929.4aff96e33c9923b5d2448f72171fd310. 2024-11-12T18:14:45,470 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1731435284929.4aff96e33c9923b5d2448f72171fd310. 2024-11-12T18:14:45,470 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1731435284929.4aff96e33c9923b5d2448f72171fd310. after waiting 0 ms 2024-11-12T18:14:45,470 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1731435284929.4aff96e33c9923b5d2448f72171fd310. 2024-11-12T18:14:45,470 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,fff,1731435284929.4aff96e33c9923b5d2448f72171fd310. 2024-11-12T18:14:45,470 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4aff96e33c9923b5d2448f72171fd310: Waiting for close lock at 1731435285470Disabling compacts and flushes for region at 1731435285470Disabling writes for close at 1731435285470Writing region close event to WAL at 1731435285470Closed at 1731435285470 2024-11-12T18:14:45,471 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => dff0a9c660bd5084b34753ea40ea3278, NAME => 'test,hhh,1731435284929.dff0a9c660bd5084b34753ea40ea3278.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d 2024-11-12T18:14:45,473 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,eee,1731435284929.c0b5ce997df05cef2b0f3fd25e49fafc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:45,473 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing c0b5ce997df05cef2b0f3fd25e49fafc, disabling compactions & flushes 2024-11-12T18:14:45,473 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,eee,1731435284929.c0b5ce997df05cef2b0f3fd25e49fafc. 2024-11-12T18:14:45,473 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1731435284929.c0b5ce997df05cef2b0f3fd25e49fafc. 2024-11-12T18:14:45,474 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1731435284929.c0b5ce997df05cef2b0f3fd25e49fafc. after waiting 0 ms 2024-11-12T18:14:45,474 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1731435284929.c0b5ce997df05cef2b0f3fd25e49fafc. 2024-11-12T18:14:45,474 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,eee,1731435284929.c0b5ce997df05cef2b0f3fd25e49fafc. 2024-11-12T18:14:45,474 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for c0b5ce997df05cef2b0f3fd25e49fafc: Waiting for close lock at 1731435285473Disabling compacts and flushes for region at 1731435285473Disabling writes for close at 1731435285474 (+1 ms)Writing region close event to WAL at 1731435285474Closed at 1731435285474 2024-11-12T18:14:45,475 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 96e9ca1050c6b52ea6556fdb4b5d5f39, NAME => 'test,iii,1731435284929.96e9ca1050c6b52ea6556fdb4b5d5f39.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d 2024-11-12T18:14:45,481 WARN [IPC Server handler 1 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-12T18:14:45,481 WARN [IPC Server handler 0 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-12T18:14:45,481 WARN [IPC Server handler 0 on default port 38437 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-12T18:14:45,481 WARN [IPC Server handler 1 on default port 38437 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-12T18:14:45,481 WARN [IPC Server handler 0 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-12T18:14:45,481 WARN [IPC Server handler 1 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-12T18:14:45,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741844_1020 (size=45) 2024-11-12T18:14:45,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741844_1020 (size=45) 2024-11-12T18:14:45,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46767 is added to blk_1073741844_1020 (size=45) 2024-11-12T18:14:45,496 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ggg,1731435284929.26fc65f8daf71d5831131b1bc6746a63.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:45,496 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 26fc65f8daf71d5831131b1bc6746a63, disabling compactions & flushes 2024-11-12T18:14:45,497 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ggg,1731435284929.26fc65f8daf71d5831131b1bc6746a63. 2024-11-12T18:14:45,497 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1731435284929.26fc65f8daf71d5831131b1bc6746a63. 2024-11-12T18:14:45,497 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1731435284929.26fc65f8daf71d5831131b1bc6746a63. after waiting 0 ms 2024-11-12T18:14:45,497 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1731435284929.26fc65f8daf71d5831131b1bc6746a63. 2024-11-12T18:14:45,497 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ggg,1731435284929.26fc65f8daf71d5831131b1bc6746a63. 2024-11-12T18:14:45,497 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 26fc65f8daf71d5831131b1bc6746a63: Waiting for close lock at 1731435285496Disabling compacts and flushes for region at 1731435285496Disabling writes for close at 1731435285497 (+1 ms)Writing region close event to WAL at 1731435285497Closed at 1731435285497 2024-11-12T18:14:45,497 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 6a6d3052a2b28af2dab58e9247bb22b7, NAME => 'test,jjj,1731435284929.6a6d3052a2b28af2dab58e9247bb22b7.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d 2024-11-12T18:14:45,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741845_1021 (size=45) 2024-11-12T18:14:45,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741845_1021 (size=45) 2024-11-12T18:14:45,502 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,iii,1731435284929.96e9ca1050c6b52ea6556fdb4b5d5f39.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:45,502 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 96e9ca1050c6b52ea6556fdb4b5d5f39, disabling compactions & flushes 2024-11-12T18:14:45,502 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,iii,1731435284929.96e9ca1050c6b52ea6556fdb4b5d5f39. 2024-11-12T18:14:45,502 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1731435284929.96e9ca1050c6b52ea6556fdb4b5d5f39. 2024-11-12T18:14:45,502 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1731435284929.96e9ca1050c6b52ea6556fdb4b5d5f39. after waiting 0 ms 2024-11-12T18:14:45,502 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1731435284929.96e9ca1050c6b52ea6556fdb4b5d5f39. 2024-11-12T18:14:45,502 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,iii,1731435284929.96e9ca1050c6b52ea6556fdb4b5d5f39. 2024-11-12T18:14:45,502 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 96e9ca1050c6b52ea6556fdb4b5d5f39: Waiting for close lock at 1731435285502Disabling compacts and flushes for region at 1731435285502Disabling writes for close at 1731435285502Writing region close event to WAL at 1731435285502Closed at 1731435285502 2024-11-12T18:14:45,503 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => d59e820eb0eef2601f5990d32a8adbc9, NAME => 'test,kkk,1731435284929.d59e820eb0eef2601f5990d32a8adbc9.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d 2024-11-12T18:14:45,510 WARN [IPC Server handler 0 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-12T18:14:45,510 WARN [IPC Server handler 0 on default port 38437 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-12T18:14:45,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741846_1022 (size=45) 2024-11-12T18:14:45,511 WARN [IPC Server handler 0 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-12T18:14:45,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741846_1022 (size=45) 2024-11-12T18:14:45,512 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,hhh,1731435284929.dff0a9c660bd5084b34753ea40ea3278.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:45,512 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing dff0a9c660bd5084b34753ea40ea3278, disabling compactions & flushes 2024-11-12T18:14:45,512 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,hhh,1731435284929.dff0a9c660bd5084b34753ea40ea3278. 2024-11-12T18:14:45,512 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1731435284929.dff0a9c660bd5084b34753ea40ea3278. 2024-11-12T18:14:45,512 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1731435284929.dff0a9c660bd5084b34753ea40ea3278. after waiting 0 ms 2024-11-12T18:14:45,512 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1731435284929.dff0a9c660bd5084b34753ea40ea3278. 2024-11-12T18:14:45,512 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,hhh,1731435284929.dff0a9c660bd5084b34753ea40ea3278. 2024-11-12T18:14:45,512 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for dff0a9c660bd5084b34753ea40ea3278: Waiting for close lock at 1731435285512Disabling compacts and flushes for region at 1731435285512Disabling writes for close at 1731435285512Writing region close event to WAL at 1731435285512Closed at 1731435285512 2024-11-12T18:14:45,513 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 394b0963ad0828345e14c3be23cb7790, NAME => 'test,lll,1731435284929.394b0963ad0828345e14c3be23cb7790.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d 2024-11-12T18:14:45,514 WARN [IPC Server handler 2 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-12T18:14:45,514 WARN [IPC Server handler 2 on default port 38437 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-12T18:14:45,514 WARN [IPC Server handler 2 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-12T18:14:45,522 WARN [IPC Server handler 0 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-12T18:14:45,522 WARN [IPC Server handler 0 on default port 38437 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-12T18:14:45,522 WARN [IPC Server handler 0 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-12T18:14:45,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741847_1023 (size=45) 2024-11-12T18:14:45,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741848_1024 (size=45) 2024-11-12T18:14:45,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741847_1023 (size=45) 2024-11-12T18:14:45,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741848_1024 (size=45) 2024-11-12T18:14:45,529 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,kkk,1731435284929.d59e820eb0eef2601f5990d32a8adbc9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:45,529 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing d59e820eb0eef2601f5990d32a8adbc9, disabling compactions & flushes 2024-11-12T18:14:45,530 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,kkk,1731435284929.d59e820eb0eef2601f5990d32a8adbc9. 2024-11-12T18:14:45,530 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,jjj,1731435284929.6a6d3052a2b28af2dab58e9247bb22b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:45,530 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1731435284929.d59e820eb0eef2601f5990d32a8adbc9. 2024-11-12T18:14:45,530 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1731435284929.d59e820eb0eef2601f5990d32a8adbc9. after waiting 0 ms 2024-11-12T18:14:45,530 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1731435284929.d59e820eb0eef2601f5990d32a8adbc9. 2024-11-12T18:14:45,530 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 6a6d3052a2b28af2dab58e9247bb22b7, disabling compactions & flushes 2024-11-12T18:14:45,530 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,kkk,1731435284929.d59e820eb0eef2601f5990d32a8adbc9. 2024-11-12T18:14:45,530 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,jjj,1731435284929.6a6d3052a2b28af2dab58e9247bb22b7. 2024-11-12T18:14:45,530 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for d59e820eb0eef2601f5990d32a8adbc9: Waiting for close lock at 1731435285529Disabling compacts and flushes for region at 1731435285529Disabling writes for close at 1731435285530 (+1 ms)Writing region close event to WAL at 1731435285530Closed at 1731435285530 2024-11-12T18:14:45,530 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1731435284929.6a6d3052a2b28af2dab58e9247bb22b7. 2024-11-12T18:14:45,530 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1731435284929.6a6d3052a2b28af2dab58e9247bb22b7. after waiting 0 ms 2024-11-12T18:14:45,530 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => c739eeba765df343af4ef63444655c9f, NAME => 'test,mmm,1731435284929.c739eeba765df343af4ef63444655c9f.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d 2024-11-12T18:14:45,530 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1731435284929.6a6d3052a2b28af2dab58e9247bb22b7. 2024-11-12T18:14:45,531 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,jjj,1731435284929.6a6d3052a2b28af2dab58e9247bb22b7. 2024-11-12T18:14:45,531 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 6a6d3052a2b28af2dab58e9247bb22b7: Waiting for close lock at 1731435285530Disabling compacts and flushes for region at 1731435285530Disabling writes for close at 1731435285530Writing region close event to WAL at 1731435285531 (+1 ms)Closed at 1731435285531 2024-11-12T18:14:45,532 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 4b703e982cd6bbf72ec2f771b120c649, NAME => 'test,nnn,1731435284929.4b703e982cd6bbf72ec2f771b120c649.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d 2024-11-12T18:14:45,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741849_1025 (size=45) 2024-11-12T18:14:45,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741849_1025 (size=45) 2024-11-12T18:14:45,536 WARN [IPC Server handler 0 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-12T18:14:45,537 WARN [IPC Server handler 0 on default port 38437 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-12T18:14:45,537 WARN [IPC Server handler 4 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-12T18:14:45,537 WARN [IPC Server handler 0 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-12T18:14:45,537 WARN [IPC Server handler 4 on default port 38437 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-12T18:14:45,537 WARN [IPC Server handler 4 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-12T18:14:45,538 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,lll,1731435284929.394b0963ad0828345e14c3be23cb7790.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:45,539 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 394b0963ad0828345e14c3be23cb7790, disabling compactions & flushes 2024-11-12T18:14:45,539 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,lll,1731435284929.394b0963ad0828345e14c3be23cb7790. 2024-11-12T18:14:45,539 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1731435284929.394b0963ad0828345e14c3be23cb7790. 2024-11-12T18:14:45,539 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1731435284929.394b0963ad0828345e14c3be23cb7790. after waiting 0 ms 2024-11-12T18:14:45,539 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1731435284929.394b0963ad0828345e14c3be23cb7790. 2024-11-12T18:14:45,539 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,lll,1731435284929.394b0963ad0828345e14c3be23cb7790. 2024-11-12T18:14:45,539 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 394b0963ad0828345e14c3be23cb7790: Waiting for close lock at 1731435285539Disabling compacts and flushes for region at 1731435285539Disabling writes for close at 1731435285539Writing region close event to WAL at 1731435285539Closed at 1731435285539 2024-11-12T18:14:45,539 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 40b076d46dbfff2e1f3c89ad5c3f814d, NAME => 'test,ooo,1731435284929.40b076d46dbfff2e1f3c89ad5c3f814d.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d 2024-11-12T18:14:45,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741851_1027 (size=45) 2024-11-12T18:14:45,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741851_1027 (size=45) 2024-11-12T18:14:45,544 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,nnn,1731435284929.4b703e982cd6bbf72ec2f771b120c649.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:45,544 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 4b703e982cd6bbf72ec2f771b120c649, disabling compactions & flushes 2024-11-12T18:14:45,544 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,nnn,1731435284929.4b703e982cd6bbf72ec2f771b120c649. 2024-11-12T18:14:45,544 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1731435284929.4b703e982cd6bbf72ec2f771b120c649. 2024-11-12T18:14:45,545 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1731435284929.4b703e982cd6bbf72ec2f771b120c649. after waiting 0 ms 2024-11-12T18:14:45,545 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1731435284929.4b703e982cd6bbf72ec2f771b120c649. 2024-11-12T18:14:45,545 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,nnn,1731435284929.4b703e982cd6bbf72ec2f771b120c649. 2024-11-12T18:14:45,545 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 4b703e982cd6bbf72ec2f771b120c649: Waiting for close lock at 1731435285544Disabling compacts and flushes for region at 1731435285544Disabling writes for close at 1731435285545 (+1 ms)Writing region close event to WAL at 1731435285545Closed at 1731435285545 2024-11-12T18:14:45,545 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 9fbe24cda440bbf727cd581ab33ad7a9, NAME => 'test,ppp,1731435284929.9fbe24cda440bbf727cd581ab33ad7a9.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d 2024-11-12T18:14:45,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741850_1026 (size=45) 2024-11-12T18:14:45,547 WARN [IPC Server handler 0 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-12T18:14:45,547 WARN [IPC Server handler 0 on default port 38437 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-12T18:14:45,547 WARN [IPC Server handler 0 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-12T18:14:45,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741850_1026 (size=45) 2024-11-12T18:14:45,550 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,mmm,1731435284929.c739eeba765df343af4ef63444655c9f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:45,550 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing c739eeba765df343af4ef63444655c9f, disabling compactions & flushes 2024-11-12T18:14:45,550 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,mmm,1731435284929.c739eeba765df343af4ef63444655c9f. 2024-11-12T18:14:45,550 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1731435284929.c739eeba765df343af4ef63444655c9f. 2024-11-12T18:14:45,550 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1731435284929.c739eeba765df343af4ef63444655c9f. after waiting 0 ms 2024-11-12T18:14:45,550 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1731435284929.c739eeba765df343af4ef63444655c9f. 2024-11-12T18:14:45,550 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,mmm,1731435284929.c739eeba765df343af4ef63444655c9f. 2024-11-12T18:14:45,550 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for c739eeba765df343af4ef63444655c9f: Waiting for close lock at 1731435285550Disabling compacts and flushes for region at 1731435285550Disabling writes for close at 1731435285550Writing region close event to WAL at 1731435285550Closed at 1731435285550 2024-11-12T18:14:45,551 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 6d7c21378c2e7788ac010db26909da83, NAME => 'test,qqq,1731435284929.6d7c21378c2e7788ac010db26909da83.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d 2024-11-12T18:14:45,555 WARN [IPC Server handler 2 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-12T18:14:45,555 WARN [IPC Server handler 2 on default port 38437 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-12T18:14:45,555 WARN [IPC Server handler 2 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-12T18:14:45,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741852_1028 (size=45) 2024-11-12T18:14:45,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741852_1028 (size=45) 2024-11-12T18:14:45,557 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ooo,1731435284929.40b076d46dbfff2e1f3c89ad5c3f814d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:45,557 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 40b076d46dbfff2e1f3c89ad5c3f814d, disabling compactions & flushes 2024-11-12T18:14:45,557 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ooo,1731435284929.40b076d46dbfff2e1f3c89ad5c3f814d. 2024-11-12T18:14:45,557 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1731435284929.40b076d46dbfff2e1f3c89ad5c3f814d. 2024-11-12T18:14:45,557 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1731435284929.40b076d46dbfff2e1f3c89ad5c3f814d. after waiting 0 ms 2024-11-12T18:14:45,557 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1731435284929.40b076d46dbfff2e1f3c89ad5c3f814d. 2024-11-12T18:14:45,557 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ooo,1731435284929.40b076d46dbfff2e1f3c89ad5c3f814d. 2024-11-12T18:14:45,557 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 40b076d46dbfff2e1f3c89ad5c3f814d: Waiting for close lock at 1731435285557Disabling compacts and flushes for region at 1731435285557Disabling writes for close at 1731435285557Writing region close event to WAL at 1731435285557Closed at 1731435285557 2024-11-12T18:14:45,558 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1b81a9be48fbe50dde523592a87a4611, NAME => 'test,rrr,1731435284929.1b81a9be48fbe50dde523592a87a4611.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d 2024-11-12T18:14:45,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741853_1029 (size=45) 2024-11-12T18:14:45,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741853_1029 (size=45) 2024-11-12T18:14:45,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46767 is added to blk_1073741853_1029 (size=45) 2024-11-12T18:14:45,566 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ppp,1731435284929.9fbe24cda440bbf727cd581ab33ad7a9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:45,566 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 9fbe24cda440bbf727cd581ab33ad7a9, disabling compactions & flushes 2024-11-12T18:14:45,566 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ppp,1731435284929.9fbe24cda440bbf727cd581ab33ad7a9. 2024-11-12T18:14:45,567 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1731435284929.9fbe24cda440bbf727cd581ab33ad7a9. 2024-11-12T18:14:45,567 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1731435284929.9fbe24cda440bbf727cd581ab33ad7a9. after waiting 0 ms 2024-11-12T18:14:45,567 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1731435284929.9fbe24cda440bbf727cd581ab33ad7a9. 2024-11-12T18:14:45,567 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ppp,1731435284929.9fbe24cda440bbf727cd581ab33ad7a9. 2024-11-12T18:14:45,567 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 9fbe24cda440bbf727cd581ab33ad7a9: Waiting for close lock at 1731435285566Disabling compacts and flushes for region at 1731435285566Disabling writes for close at 1731435285567 (+1 ms)Writing region close event to WAL at 1731435285567Closed at 1731435285567 2024-11-12T18:14:45,567 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 4795f188f4931ea02df11d7474e4daa4, NAME => 'test,sss,1731435284929.4795f188f4931ea02df11d7474e4daa4.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d 2024-11-12T18:14:45,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741854_1030 (size=45) 2024-11-12T18:14:45,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741854_1030 (size=45) 2024-11-12T18:14:45,577 WARN [IPC Server handler 0 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-12T18:14:45,577 WARN [IPC Server handler 0 on default port 38437 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-12T18:14:45,577 WARN [IPC Server handler 0 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-12T18:14:45,577 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,qqq,1731435284929.6d7c21378c2e7788ac010db26909da83.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:45,577 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 6d7c21378c2e7788ac010db26909da83, disabling compactions & flushes 2024-11-12T18:14:45,577 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,qqq,1731435284929.6d7c21378c2e7788ac010db26909da83. 2024-11-12T18:14:45,577 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1731435284929.6d7c21378c2e7788ac010db26909da83. 2024-11-12T18:14:45,577 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1731435284929.6d7c21378c2e7788ac010db26909da83. after waiting 0 ms 2024-11-12T18:14:45,577 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1731435284929.6d7c21378c2e7788ac010db26909da83. 2024-11-12T18:14:45,578 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,qqq,1731435284929.6d7c21378c2e7788ac010db26909da83. 2024-11-12T18:14:45,578 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 6d7c21378c2e7788ac010db26909da83: Waiting for close lock at 1731435285577Disabling compacts and flushes for region at 1731435285577Disabling writes for close at 1731435285577Writing region close event to WAL at 1731435285578 (+1 ms)Closed at 1731435285578 2024-11-12T18:14:45,578 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 4e38ac98b32f350d84df37b6915c56a3, NAME => 'test,ttt,1731435284929.4e38ac98b32f350d84df37b6915c56a3.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d 2024-11-12T18:14:45,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741856_1032 (size=45) 2024-11-12T18:14:45,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741856_1032 (size=45) 2024-11-12T18:14:45,587 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,sss,1731435284929.4795f188f4931ea02df11d7474e4daa4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:45,588 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 4795f188f4931ea02df11d7474e4daa4, disabling compactions & flushes 2024-11-12T18:14:45,588 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,sss,1731435284929.4795f188f4931ea02df11d7474e4daa4. 2024-11-12T18:14:45,588 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1731435284929.4795f188f4931ea02df11d7474e4daa4. 2024-11-12T18:14:45,588 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1731435284929.4795f188f4931ea02df11d7474e4daa4. after waiting 0 ms 2024-11-12T18:14:45,588 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1731435284929.4795f188f4931ea02df11d7474e4daa4. 2024-11-12T18:14:45,588 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,sss,1731435284929.4795f188f4931ea02df11d7474e4daa4. 2024-11-12T18:14:45,588 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 4795f188f4931ea02df11d7474e4daa4: Waiting for close lock at 1731435285588Disabling compacts and flushes for region at 1731435285588Disabling writes for close at 1731435285588Writing region close event to WAL at 1731435285588Closed at 1731435285588 2024-11-12T18:14:45,589 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => a7b0558136890d45b56827e5af943cfa, NAME => 'test,uuu,1731435284929.a7b0558136890d45b56827e5af943cfa.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d 2024-11-12T18:14:45,590 WARN [IPC Server handler 1 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-12T18:14:45,590 WARN [IPC Server handler 1 on default port 38437 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-12T18:14:45,590 WARN [IPC Server handler 1 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-12T18:14:45,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741855_1031 (size=45) 2024-11-12T18:14:45,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741855_1031 (size=45) 2024-11-12T18:14:45,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46767 is added to blk_1073741855_1031 (size=45) 2024-11-12T18:14:45,597 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,rrr,1731435284929.1b81a9be48fbe50dde523592a87a4611.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:45,598 WARN [IPC Server handler 4 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-12T18:14:45,598 WARN [IPC Server handler 4 on default port 38437 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-12T18:14:45,598 WARN [IPC Server handler 4 on default port 38437 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-12T18:14:45,599 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 1b81a9be48fbe50dde523592a87a4611, disabling compactions & flushes 2024-11-12T18:14:45,599 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,rrr,1731435284929.1b81a9be48fbe50dde523592a87a4611. 2024-11-12T18:14:45,600 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1731435284929.1b81a9be48fbe50dde523592a87a4611. 2024-11-12T18:14:45,600 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1731435284929.1b81a9be48fbe50dde523592a87a4611. after waiting 0 ms 2024-11-12T18:14:45,600 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1731435284929.1b81a9be48fbe50dde523592a87a4611. 2024-11-12T18:14:45,600 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,rrr,1731435284929.1b81a9be48fbe50dde523592a87a4611. 2024-11-12T18:14:45,600 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1b81a9be48fbe50dde523592a87a4611: Waiting for close lock at 1731435285599Disabling compacts and flushes for region at 1731435285599Disabling writes for close at 1731435285600 (+1 ms)Writing region close event to WAL at 1731435285600Closed at 1731435285600 2024-11-12T18:14:45,600 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 56e168ba1a3103978bee5b6c43feba36, NAME => 'test,vvv,1731435284929.56e168ba1a3103978bee5b6c43feba36.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d 2024-11-12T18:14:45,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741857_1033 (size=45) 2024-11-12T18:14:45,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741857_1033 (size=45) 2024-11-12T18:14:45,608 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ttt,1731435284929.4e38ac98b32f350d84df37b6915c56a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:45,608 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 4e38ac98b32f350d84df37b6915c56a3, disabling compactions & flushes 2024-11-12T18:14:45,608 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ttt,1731435284929.4e38ac98b32f350d84df37b6915c56a3. 2024-11-12T18:14:45,608 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1731435284929.4e38ac98b32f350d84df37b6915c56a3. 2024-11-12T18:14:45,608 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1731435284929.4e38ac98b32f350d84df37b6915c56a3. after waiting 0 ms 2024-11-12T18:14:45,608 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1731435284929.4e38ac98b32f350d84df37b6915c56a3. 2024-11-12T18:14:45,608 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ttt,1731435284929.4e38ac98b32f350d84df37b6915c56a3. 2024-11-12T18:14:45,608 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 4e38ac98b32f350d84df37b6915c56a3: Waiting for close lock at 1731435285608Disabling compacts and flushes for region at 1731435285608Disabling writes for close at 1731435285608Writing region close event to WAL at 1731435285608Closed at 1731435285608 2024-11-12T18:14:45,609 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => ec76766d593d312811f043efb129a5a3, NAME => 'test,www,1731435284929.ec76766d593d312811f043efb129a5a3.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d 2024-11-12T18:14:45,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741858_1034 (size=45) 2024-11-12T18:14:45,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46767 is added to blk_1073741858_1034 (size=45) 2024-11-12T18:14:45,614 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,uuu,1731435284929.a7b0558136890d45b56827e5af943cfa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:45,614 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing a7b0558136890d45b56827e5af943cfa, disabling compactions & flushes 2024-11-12T18:14:45,614 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,uuu,1731435284929.a7b0558136890d45b56827e5af943cfa. 2024-11-12T18:14:45,614 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1731435284929.a7b0558136890d45b56827e5af943cfa. 2024-11-12T18:14:45,614 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1731435284929.a7b0558136890d45b56827e5af943cfa. after waiting 0 ms 2024-11-12T18:14:45,614 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1731435284929.a7b0558136890d45b56827e5af943cfa. 2024-11-12T18:14:45,614 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,uuu,1731435284929.a7b0558136890d45b56827e5af943cfa. 2024-11-12T18:14:45,614 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for a7b0558136890d45b56827e5af943cfa: Waiting for close lock at 1731435285614Disabling compacts and flushes for region at 1731435285614Disabling writes for close at 1731435285614Writing region close event to WAL at 1731435285614Closed at 1731435285614 2024-11-12T18:14:45,614 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 0178ff26f4e2d0ab40d50fe6fcd026be, NAME => 'test,xxx,1731435284929.0178ff26f4e2d0ab40d50fe6fcd026be.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d 2024-11-12T18:14:45,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T18:14:45,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741859_1035 (size=45) 2024-11-12T18:14:45,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741859_1035 (size=45) 2024-11-12T18:14:45,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46767 is added to blk_1073741859_1035 (size=45) 2024-11-12T18:14:45,621 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,vvv,1731435284929.56e168ba1a3103978bee5b6c43feba36.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:45,622 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 56e168ba1a3103978bee5b6c43feba36, disabling compactions & flushes 2024-11-12T18:14:45,622 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,vvv,1731435284929.56e168ba1a3103978bee5b6c43feba36. 2024-11-12T18:14:45,622 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1731435284929.56e168ba1a3103978bee5b6c43feba36. 2024-11-12T18:14:45,622 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1731435284929.56e168ba1a3103978bee5b6c43feba36. after waiting 0 ms 2024-11-12T18:14:45,622 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1731435284929.56e168ba1a3103978bee5b6c43feba36. 2024-11-12T18:14:45,622 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,vvv,1731435284929.56e168ba1a3103978bee5b6c43feba36. 2024-11-12T18:14:45,622 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 56e168ba1a3103978bee5b6c43feba36: Waiting for close lock at 1731435285622Disabling compacts and flushes for region at 1731435285622Disabling writes for close at 1731435285622Writing region close event to WAL at 1731435285622Closed at 1731435285622 2024-11-12T18:14:45,623 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fde2b7fbe9fa7c8a87ab544f5542d82a, NAME => 'test,yyy,1731435284929.fde2b7fbe9fa7c8a87ab544f5542d82a.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d 2024-11-12T18:14:45,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741860_1036 (size=45) 2024-11-12T18:14:45,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741860_1036 (size=45) 2024-11-12T18:14:45,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46767 is added to blk_1073741860_1036 (size=45) 2024-11-12T18:14:45,629 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,www,1731435284929.ec76766d593d312811f043efb129a5a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:45,630 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing ec76766d593d312811f043efb129a5a3, disabling compactions & flushes 2024-11-12T18:14:45,630 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,www,1731435284929.ec76766d593d312811f043efb129a5a3. 2024-11-12T18:14:45,630 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1731435284929.ec76766d593d312811f043efb129a5a3. 2024-11-12T18:14:45,630 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1731435284929.ec76766d593d312811f043efb129a5a3. after waiting 0 ms 2024-11-12T18:14:45,630 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1731435284929.ec76766d593d312811f043efb129a5a3. 2024-11-12T18:14:45,630 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,www,1731435284929.ec76766d593d312811f043efb129a5a3. 2024-11-12T18:14:45,630 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for ec76766d593d312811f043efb129a5a3: Waiting for close lock at 1731435285630Disabling compacts and flushes for region at 1731435285630Disabling writes for close at 1731435285630Writing region close event to WAL at 1731435285630Closed at 1731435285630 2024-11-12T18:14:45,631 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => e4348dabb8addd0673ec5266827d8b4d, NAME => 'test,zzz,1731435284929.e4348dabb8addd0673ec5266827d8b4d.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d 2024-11-12T18:14:45,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46767 is added to blk_1073741861_1037 (size=45) 2024-11-12T18:14:45,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741861_1037 (size=45) 2024-11-12T18:14:45,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741861_1037 (size=45) 2024-11-12T18:14:45,634 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,xxx,1731435284929.0178ff26f4e2d0ab40d50fe6fcd026be.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:45,634 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 0178ff26f4e2d0ab40d50fe6fcd026be, disabling compactions & flushes 2024-11-12T18:14:45,634 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,xxx,1731435284929.0178ff26f4e2d0ab40d50fe6fcd026be. 2024-11-12T18:14:45,634 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1731435284929.0178ff26f4e2d0ab40d50fe6fcd026be. 2024-11-12T18:14:45,635 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1731435284929.0178ff26f4e2d0ab40d50fe6fcd026be. after waiting 0 ms 2024-11-12T18:14:45,635 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1731435284929.0178ff26f4e2d0ab40d50fe6fcd026be. 2024-11-12T18:14:45,635 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,xxx,1731435284929.0178ff26f4e2d0ab40d50fe6fcd026be. 2024-11-12T18:14:45,635 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 0178ff26f4e2d0ab40d50fe6fcd026be: Waiting for close lock at 1731435285634Disabling compacts and flushes for region at 1731435285634Disabling writes for close at 1731435285635 (+1 ms)Writing region close event to WAL at 1731435285635Closed at 1731435285635 2024-11-12T18:14:45,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741862_1038 (size=45) 2024-11-12T18:14:45,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46767 is added to blk_1073741862_1038 (size=45) 2024-11-12T18:14:45,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741862_1038 (size=45) 2024-11-12T18:14:45,640 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,yyy,1731435284929.fde2b7fbe9fa7c8a87ab544f5542d82a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:45,640 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing fde2b7fbe9fa7c8a87ab544f5542d82a, disabling compactions & flushes 2024-11-12T18:14:45,640 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,yyy,1731435284929.fde2b7fbe9fa7c8a87ab544f5542d82a. 2024-11-12T18:14:45,640 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1731435284929.fde2b7fbe9fa7c8a87ab544f5542d82a. 2024-11-12T18:14:45,640 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1731435284929.fde2b7fbe9fa7c8a87ab544f5542d82a. after waiting 0 ms 2024-11-12T18:14:45,640 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1731435284929.fde2b7fbe9fa7c8a87ab544f5542d82a. 2024-11-12T18:14:45,640 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,yyy,1731435284929.fde2b7fbe9fa7c8a87ab544f5542d82a. 2024-11-12T18:14:45,640 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for fde2b7fbe9fa7c8a87ab544f5542d82a: Waiting for close lock at 1731435285640Disabling compacts and flushes for region at 1731435285640Disabling writes for close at 1731435285640Writing region close event to WAL at 1731435285640Closed at 1731435285640 2024-11-12T18:14:45,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741863_1039 (size=42) 2024-11-12T18:14:45,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46767 is added to blk_1073741863_1039 (size=42) 2024-11-12T18:14:45,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741863_1039 (size=42) 2024-11-12T18:14:45,645 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,zzz,1731435284929.e4348dabb8addd0673ec5266827d8b4d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:45,645 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing e4348dabb8addd0673ec5266827d8b4d, disabling compactions & flushes 2024-11-12T18:14:45,645 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,zzz,1731435284929.e4348dabb8addd0673ec5266827d8b4d. 2024-11-12T18:14:45,645 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1731435284929.e4348dabb8addd0673ec5266827d8b4d. 2024-11-12T18:14:45,645 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1731435284929.e4348dabb8addd0673ec5266827d8b4d. after waiting 0 ms 2024-11-12T18:14:45,645 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1731435284929.e4348dabb8addd0673ec5266827d8b4d. 2024-11-12T18:14:45,645 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,zzz,1731435284929.e4348dabb8addd0673ec5266827d8b4d. 2024-11-12T18:14:45,645 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for e4348dabb8addd0673ec5266827d8b4d: Waiting for close lock at 1731435285645Disabling compacts and flushes for region at 1731435285645Disabling writes for close at 1731435285645Writing region close event to WAL at 1731435285645Closed at 1731435285645 2024-11-12T18:14:45,648 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T18:14:45,653 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1731435284929.2248a0d1296d1dbe4f8c9805798db03c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731435285648"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731435285648"}]},"ts":"1731435285648"} 2024-11-12T18:14:45,653 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1731435284929.3a62a194de5221e9f84b2cfd2e83a46a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731435285648"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731435285648"}]},"ts":"1731435285648"} 2024-11-12T18:14:45,653 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1731435284929.5dcfe82d27e00f564e4ad71ea2e78f5c.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731435285648"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731435285648"}]},"ts":"1731435285648"} 2024-11-12T18:14:45,654 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1731435284929.6e290ad15dec09a0cb11782988af7c4c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731435285648"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731435285648"}]},"ts":"1731435285648"} 2024-11-12T18:14:45,654 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1731435284929.4aff96e33c9923b5d2448f72171fd310.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731435285648"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731435285648"}]},"ts":"1731435285648"} 2024-11-12T18:14:45,654 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1731435284929.c0b5ce997df05cef2b0f3fd25e49fafc.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731435285648"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731435285648"}]},"ts":"1731435285648"} 2024-11-12T18:14:45,654 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1731435284929.26fc65f8daf71d5831131b1bc6746a63.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731435285648"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731435285648"}]},"ts":"1731435285648"} 2024-11-12T18:14:45,654 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1731435284929.96e9ca1050c6b52ea6556fdb4b5d5f39.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731435285648"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731435285648"}]},"ts":"1731435285648"} 2024-11-12T18:14:45,654 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1731435284929.dff0a9c660bd5084b34753ea40ea3278.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731435285648"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731435285648"}]},"ts":"1731435285648"} 2024-11-12T18:14:45,654 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1731435284929.d59e820eb0eef2601f5990d32a8adbc9.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731435285648"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731435285648"}]},"ts":"1731435285648"} 2024-11-12T18:14:45,654 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1731435284929.6a6d3052a2b28af2dab58e9247bb22b7.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731435285648"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731435285648"}]},"ts":"1731435285648"} 2024-11-12T18:14:45,655 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1731435284929.394b0963ad0828345e14c3be23cb7790.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731435285648"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731435285648"}]},"ts":"1731435285648"} 2024-11-12T18:14:45,655 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1731435284929.4b703e982cd6bbf72ec2f771b120c649.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731435285648"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731435285648"}]},"ts":"1731435285648"} 2024-11-12T18:14:45,655 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1731435284929.c739eeba765df343af4ef63444655c9f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731435285648"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731435285648"}]},"ts":"1731435285648"} 2024-11-12T18:14:45,655 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1731435284929.40b076d46dbfff2e1f3c89ad5c3f814d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731435285648"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731435285648"}]},"ts":"1731435285648"} 2024-11-12T18:14:45,655 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1731435284929.9fbe24cda440bbf727cd581ab33ad7a9.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731435285648"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731435285648"}]},"ts":"1731435285648"} 2024-11-12T18:14:45,655 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1731435284929.6d7c21378c2e7788ac010db26909da83.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731435285648"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731435285648"}]},"ts":"1731435285648"} 2024-11-12T18:14:45,655 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1731435284929.4795f188f4931ea02df11d7474e4daa4.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731435285648"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731435285648"}]},"ts":"1731435285648"} 2024-11-12T18:14:45,655 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1731435284929.1b81a9be48fbe50dde523592a87a4611.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731435285648"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731435285648"}]},"ts":"1731435285648"} 2024-11-12T18:14:45,655 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1731435284929.4e38ac98b32f350d84df37b6915c56a3.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731435285648"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731435285648"}]},"ts":"1731435285648"} 2024-11-12T18:14:45,656 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1731435284929.a7b0558136890d45b56827e5af943cfa.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731435285648"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731435285648"}]},"ts":"1731435285648"} 2024-11-12T18:14:45,656 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1731435284929.56e168ba1a3103978bee5b6c43feba36.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731435285648"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731435285648"}]},"ts":"1731435285648"} 2024-11-12T18:14:45,656 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1731435284929.ec76766d593d312811f043efb129a5a3.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731435285648"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731435285648"}]},"ts":"1731435285648"} 2024-11-12T18:14:45,656 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1731435284929.0178ff26f4e2d0ab40d50fe6fcd026be.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731435285648"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731435285648"}]},"ts":"1731435285648"} 2024-11-12T18:14:45,656 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1731435284929.fde2b7fbe9fa7c8a87ab544f5542d82a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731435285648"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731435285648"}]},"ts":"1731435285648"} 2024-11-12T18:14:45,656 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1731435284929.e4348dabb8addd0673ec5266827d8b4d.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731435285648"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731435285648"}]},"ts":"1731435285648"} 2024-11-12T18:14:45,701 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-12T18:14:45,704 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T18:14:45,707 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731435285704"}]},"ts":"1731435285704"} 2024-11-12T18:14:45,712 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-11-12T18:14:45,713 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {9911683f163c=0} racks are {/default-rack=0} 2024-11-12T18:14:45,730 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-12T18:14:45,730 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-12T18:14:45,730 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-12T18:14:45,730 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-12T18:14:45,730 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-12T18:14:45,730 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-12T18:14:45,730 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-12T18:14:45,730 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-12T18:14:45,730 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-12T18:14:45,730 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-12T18:14:45,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5dcfe82d27e00f564e4ad71ea2e78f5c, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3a62a194de5221e9f84b2cfd2e83a46a, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6e290ad15dec09a0cb11782988af7c4c, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c0b5ce997df05cef2b0f3fd25e49fafc, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4aff96e33c9923b5d2448f72171fd310, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=dff0a9c660bd5084b34753ea40ea3278, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=96e9ca1050c6b52ea6556fdb4b5d5f39, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6a6d3052a2b28af2dab58e9247bb22b7, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=394b0963ad0828345e14c3be23cb7790, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c739eeba765df343af4ef63444655c9f, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=40b076d46dbfff2e1f3c89ad5c3f814d, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6d7c21378c2e7788ac010db26909da83, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4795f188f4931ea02df11d7474e4daa4, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4e38ac98b32f350d84df37b6915c56a3, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a7b0558136890d45b56827e5af943cfa, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ec76766d593d312811f043efb129a5a3, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0178ff26f4e2d0ab40d50fe6fcd026be, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e4348dabb8addd0673ec5266827d8b4d, ASSIGN}] 2024-11-12T18:14:45,744 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e4348dabb8addd0673ec5266827d8b4d, ASSIGN 2024-11-12T18:14:45,745 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e4348dabb8addd0673ec5266827d8b4d, ASSIGN; state=OFFLINE, location=9911683f163c,34847,1731435282512; forceNewPlan=false, retain=false 2024-11-12T18:14:45,747 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN 2024-11-12T18:14:45,748 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN; state=OFFLINE, location=9911683f163c,40543,1731435282322; forceNewPlan=false, retain=false 2024-11-12T18:14:45,750 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0178ff26f4e2d0ab40d50fe6fcd026be, ASSIGN 2024-11-12T18:14:45,751 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0178ff26f4e2d0ab40d50fe6fcd026be, ASSIGN; state=OFFLINE, location=9911683f163c,45243,1731435282573; forceNewPlan=false, retain=false 2024-11-12T18:14:45,752 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ec76766d593d312811f043efb129a5a3, ASSIGN 2024-11-12T18:14:45,754 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ec76766d593d312811f043efb129a5a3, ASSIGN; state=OFFLINE, location=9911683f163c,34847,1731435282512; forceNewPlan=false, retain=false 2024-11-12T18:14:45,755 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN 2024-11-12T18:14:45,757 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN; state=OFFLINE, location=9911683f163c,40543,1731435282322; forceNewPlan=false, retain=false 2024-11-12T18:14:45,758 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a7b0558136890d45b56827e5af943cfa, ASSIGN 2024-11-12T18:14:45,759 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a7b0558136890d45b56827e5af943cfa, ASSIGN; state=OFFLINE, location=9911683f163c,45243,1731435282573; forceNewPlan=false, retain=false 2024-11-12T18:14:45,760 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4e38ac98b32f350d84df37b6915c56a3, ASSIGN 2024-11-12T18:14:45,762 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4e38ac98b32f350d84df37b6915c56a3, ASSIGN; state=OFFLINE, location=9911683f163c,34847,1731435282512; forceNewPlan=false, retain=false 2024-11-12T18:14:45,763 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4795f188f4931ea02df11d7474e4daa4, ASSIGN 2024-11-12T18:14:45,764 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4795f188f4931ea02df11d7474e4daa4, ASSIGN; state=OFFLINE, location=9911683f163c,45243,1731435282573; forceNewPlan=false, retain=false 2024-11-12T18:14:45,765 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN 2024-11-12T18:14:45,767 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN; state=OFFLINE, location=9911683f163c,40543,1731435282322; forceNewPlan=false, retain=false 2024-11-12T18:14:45,768 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6d7c21378c2e7788ac010db26909da83, ASSIGN 2024-11-12T18:14:45,770 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6d7c21378c2e7788ac010db26909da83, ASSIGN; state=OFFLINE, location=9911683f163c,34847,1731435282512; forceNewPlan=false, retain=false 2024-11-12T18:14:45,772 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN 2024-11-12T18:14:45,773 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN; state=OFFLINE, location=9911683f163c,40543,1731435282322; forceNewPlan=false, retain=false 2024-11-12T18:14:45,774 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=40b076d46dbfff2e1f3c89ad5c3f814d, ASSIGN 2024-11-12T18:14:45,776 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=40b076d46dbfff2e1f3c89ad5c3f814d, ASSIGN; state=OFFLINE, location=9911683f163c,45243,1731435282573; forceNewPlan=false, retain=false 2024-11-12T18:14:45,777 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN 2024-11-12T18:14:45,778 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN; state=OFFLINE, location=9911683f163c,40543,1731435282322; forceNewPlan=false, retain=false 2024-11-12T18:14:45,780 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c739eeba765df343af4ef63444655c9f, ASSIGN 2024-11-12T18:14:45,781 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c739eeba765df343af4ef63444655c9f, ASSIGN; state=OFFLINE, location=9911683f163c,34847,1731435282512; forceNewPlan=false, retain=false 2024-11-12T18:14:45,782 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=394b0963ad0828345e14c3be23cb7790, ASSIGN 2024-11-12T18:14:45,784 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=394b0963ad0828345e14c3be23cb7790, ASSIGN; state=OFFLINE, location=9911683f163c,45243,1731435282573; forceNewPlan=false, retain=false 2024-11-12T18:14:45,785 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN 2024-11-12T18:14:45,787 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN; state=OFFLINE, location=9911683f163c,40543,1731435282322; forceNewPlan=false, retain=false 2024-11-12T18:14:45,788 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6a6d3052a2b28af2dab58e9247bb22b7, ASSIGN 2024-11-12T18:14:45,790 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6a6d3052a2b28af2dab58e9247bb22b7, ASSIGN; state=OFFLINE, location=9911683f163c,34847,1731435282512; forceNewPlan=false, retain=false 2024-11-12T18:14:45,791 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=96e9ca1050c6b52ea6556fdb4b5d5f39, ASSIGN 2024-11-12T18:14:45,793 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=96e9ca1050c6b52ea6556fdb4b5d5f39, ASSIGN; state=OFFLINE, location=9911683f163c,34847,1731435282512; forceNewPlan=false, retain=false 2024-11-12T18:14:45,794 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=dff0a9c660bd5084b34753ea40ea3278, ASSIGN 2024-11-12T18:14:45,796 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=dff0a9c660bd5084b34753ea40ea3278, ASSIGN; state=OFFLINE, location=9911683f163c,45243,1731435282573; forceNewPlan=false, retain=false 2024-11-12T18:14:45,797 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN 2024-11-12T18:14:45,798 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN; state=OFFLINE, location=9911683f163c,40543,1731435282322; forceNewPlan=false, retain=false 2024-11-12T18:14:45,799 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4aff96e33c9923b5d2448f72171fd310, ASSIGN 2024-11-12T18:14:45,801 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4aff96e33c9923b5d2448f72171fd310, ASSIGN; state=OFFLINE, location=9911683f163c,34847,1731435282512; forceNewPlan=false, retain=false 2024-11-12T18:14:45,802 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c0b5ce997df05cef2b0f3fd25e49fafc, ASSIGN 2024-11-12T18:14:45,803 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c0b5ce997df05cef2b0f3fd25e49fafc, ASSIGN; state=OFFLINE, location=9911683f163c,45243,1731435282573; forceNewPlan=false, retain=false 2024-11-12T18:14:45,805 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6e290ad15dec09a0cb11782988af7c4c, ASSIGN 2024-11-12T18:14:45,806 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6e290ad15dec09a0cb11782988af7c4c, ASSIGN; state=OFFLINE, location=9911683f163c,40543,1731435282322; forceNewPlan=false, retain=false 2024-11-12T18:14:45,809 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3a62a194de5221e9f84b2cfd2e83a46a, ASSIGN 2024-11-12T18:14:45,810 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=3a62a194de5221e9f84b2cfd2e83a46a, ASSIGN; state=OFFLINE, location=9911683f163c,34847,1731435282512; forceNewPlan=false, retain=false 2024-11-12T18:14:45,811 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN 2024-11-12T18:14:45,812 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN; state=OFFLINE, location=9911683f163c,40543,1731435282322; forceNewPlan=false, retain=false 2024-11-12T18:14:45,813 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5dcfe82d27e00f564e4ad71ea2e78f5c, ASSIGN 2024-11-12T18:14:45,815 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5dcfe82d27e00f564e4ad71ea2e78f5c, ASSIGN; state=OFFLINE, location=9911683f163c,45243,1731435282573; forceNewPlan=false, retain=false 2024-11-12T18:14:45,899 INFO [9911683f163c:36779 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-12T18:14:45,900 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=0178ff26f4e2d0ab40d50fe6fcd026be, regionState=OPENING, regionLocation=9911683f163c,45243,1731435282573 2024-11-12T18:14:45,901 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=a7b0558136890d45b56827e5af943cfa, regionState=OPENING, regionLocation=9911683f163c,45243,1731435282573 2024-11-12T18:14:45,902 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=4795f188f4931ea02df11d7474e4daa4, regionState=OPENING, regionLocation=9911683f163c,45243,1731435282573 2024-11-12T18:14:45,902 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=40b076d46dbfff2e1f3c89ad5c3f814d, regionState=OPENING, regionLocation=9911683f163c,45243,1731435282573 2024-11-12T18:14:45,903 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=394b0963ad0828345e14c3be23cb7790, regionState=OPENING, regionLocation=9911683f163c,45243,1731435282573 2024-11-12T18:14:45,904 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=dff0a9c660bd5084b34753ea40ea3278, regionState=OPENING, regionLocation=9911683f163c,45243,1731435282573 2024-11-12T18:14:45,904 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=c0b5ce997df05cef2b0f3fd25e49fafc, regionState=OPENING, regionLocation=9911683f163c,45243,1731435282573 2024-11-12T18:14:45,904 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0178ff26f4e2d0ab40d50fe6fcd026be, ASSIGN because future has completed 2024-11-12T18:14:45,905 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5dcfe82d27e00f564e4ad71ea2e78f5c, regionState=OPENING, regionLocation=9911683f163c,45243,1731435282573 2024-11-12T18:14:45,906 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0178ff26f4e2d0ab40d50fe6fcd026be, server=9911683f163c,45243,1731435282573}] 2024-11-12T18:14:45,910 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a7b0558136890d45b56827e5af943cfa, ASSIGN because future has completed 2024-11-12T18:14:45,912 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7b0558136890d45b56827e5af943cfa, server=9911683f163c,45243,1731435282573}] 2024-11-12T18:14:45,912 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4795f188f4931ea02df11d7474e4daa4, ASSIGN because future has completed 2024-11-12T18:14:45,913 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=40b076d46dbfff2e1f3c89ad5c3f814d, ASSIGN because future has completed 2024-11-12T18:14:45,915 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=394b0963ad0828345e14c3be23cb7790, ASSIGN because future has completed 2024-11-12T18:14:45,916 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=dff0a9c660bd5084b34753ea40ea3278, ASSIGN because future has completed 2024-11-12T18:14:45,918 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure dff0a9c660bd5084b34753ea40ea3278, server=9911683f163c,45243,1731435282573}] 2024-11-12T18:14:45,918 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c0b5ce997df05cef2b0f3fd25e49fafc, ASSIGN because future has completed 2024-11-12T18:14:45,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5dcfe82d27e00f564e4ad71ea2e78f5c, ASSIGN because future has completed 2024-11-12T18:14:45,924 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5dcfe82d27e00f564e4ad71ea2e78f5c, server=9911683f163c,45243,1731435282573}] 2024-11-12T18:14:45,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0b5ce997df05cef2b0f3fd25e49fafc, server=9911683f163c,45243,1731435282573}] 2024-11-12T18:14:45,934 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 394b0963ad0828345e14c3be23cb7790, server=9911683f163c,45243,1731435282573}] 2024-11-12T18:14:45,938 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 40b076d46dbfff2e1f3c89ad5c3f814d, server=9911683f163c,45243,1731435282573}] 2024-11-12T18:14:45,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4795f188f4931ea02df11d7474e4daa4, server=9911683f163c,45243,1731435282573}] 2024-11-12T18:14:45,948 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=fde2b7fbe9fa7c8a87ab544f5542d82a, regionState=OPENING, regionLocation=9911683f163c,40543,1731435282322 2024-11-12T18:14:45,948 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=56e168ba1a3103978bee5b6c43feba36, regionState=OPENING, regionLocation=9911683f163c,40543,1731435282322 2024-11-12T18:14:45,949 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=1b81a9be48fbe50dde523592a87a4611, regionState=OPENING, regionLocation=9911683f163c,40543,1731435282322 2024-11-12T18:14:45,949 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=9fbe24cda440bbf727cd581ab33ad7a9, regionState=OPENING, regionLocation=9911683f163c,40543,1731435282322 2024-11-12T18:14:45,950 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=4b703e982cd6bbf72ec2f771b120c649, regionState=OPENING, regionLocation=9911683f163c,40543,1731435282322 2024-11-12T18:14:45,950 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=d59e820eb0eef2601f5990d32a8adbc9, regionState=OPENING, regionLocation=9911683f163c,40543,1731435282322 2024-11-12T18:14:45,951 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=26fc65f8daf71d5831131b1bc6746a63, regionState=OPENING, regionLocation=9911683f163c,40543,1731435282322 2024-11-12T18:14:45,951 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN because future has completed 2024-11-12T18:14:45,952 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure fde2b7fbe9fa7c8a87ab544f5542d82a, server=9911683f163c,40543,1731435282322}] 2024-11-12T18:14:45,953 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN because future has completed 2024-11-12T18:14:45,954 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN because future has completed 2024-11-12T18:14:45,956 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN because future has completed 2024-11-12T18:14:45,957 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN because future has completed 2024-11-12T18:14:45,958 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,40543,1731435282322}] 2024-11-12T18:14:45,959 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN because future has completed 2024-11-12T18:14:45,960 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN because future has completed 2024-11-12T18:14:45,963 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26fc65f8daf71d5831131b1bc6746a63, server=9911683f163c,40543,1731435282322}] 2024-11-12T18:14:45,967 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure d59e820eb0eef2601f5990d32a8adbc9, server=9911683f163c,40543,1731435282322}] 2024-11-12T18:14:45,972 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9fbe24cda440bbf727cd581ab33ad7a9, server=9911683f163c,40543,1731435282322}] 2024-11-12T18:14:45,976 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b81a9be48fbe50dde523592a87a4611, server=9911683f163c,40543,1731435282322}] 2024-11-12T18:14:45,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 56e168ba1a3103978bee5b6c43feba36, server=9911683f163c,40543,1731435282322}] 2024-11-12T18:14:45,985 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=6e290ad15dec09a0cb11782988af7c4c, regionState=OPENING, regionLocation=9911683f163c,40543,1731435282322 2024-11-12T18:14:45,986 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=2248a0d1296d1dbe4f8c9805798db03c, regionState=OPENING, regionLocation=9911683f163c,40543,1731435282322 2024-11-12T18:14:45,986 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=e4348dabb8addd0673ec5266827d8b4d, regionState=OPENING, regionLocation=9911683f163c,34847,1731435282512 2024-11-12T18:14:45,987 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=ec76766d593d312811f043efb129a5a3, regionState=OPENING, regionLocation=9911683f163c,34847,1731435282512 2024-11-12T18:14:45,987 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=4e38ac98b32f350d84df37b6915c56a3, regionState=OPENING, regionLocation=9911683f163c,34847,1731435282512 2024-11-12T18:14:45,988 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=6d7c21378c2e7788ac010db26909da83, regionState=OPENING, regionLocation=9911683f163c,34847,1731435282512 2024-11-12T18:14:45,988 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6e290ad15dec09a0cb11782988af7c4c, ASSIGN because future has completed 2024-11-12T18:14:45,989 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6e290ad15dec09a0cb11782988af7c4c, server=9911683f163c,40543,1731435282322}] 2024-11-12T18:14:45,990 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN because future has completed 2024-11-12T18:14:45,991 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e4348dabb8addd0673ec5266827d8b4d, ASSIGN because future has completed 2024-11-12T18:14:45,993 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ec76766d593d312811f043efb129a5a3, ASSIGN because future has completed 2024-11-12T18:14:45,994 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure ec76766d593d312811f043efb129a5a3, server=9911683f163c,34847,1731435282512}] 2024-11-12T18:14:45,994 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4e38ac98b32f350d84df37b6915c56a3, ASSIGN because future has completed 2024-11-12T18:14:45,995 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6d7c21378c2e7788ac010db26909da83, ASSIGN because future has completed 2024-11-12T18:14:45,998 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d7c21378c2e7788ac010db26909da83, server=9911683f163c,34847,1731435282512}] 2024-11-12T18:14:46,002 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e38ac98b32f350d84df37b6915c56a3, server=9911683f163c,34847,1731435282512}] 2024-11-12T18:14:46,006 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure e4348dabb8addd0673ec5266827d8b4d, server=9911683f163c,34847,1731435282512}] 2024-11-12T18:14:46,011 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2248a0d1296d1dbe4f8c9805798db03c, server=9911683f163c,40543,1731435282322}] 2024-11-12T18:14:46,016 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=c739eeba765df343af4ef63444655c9f, regionState=OPENING, regionLocation=9911683f163c,34847,1731435282512 2024-11-12T18:14:46,016 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=6a6d3052a2b28af2dab58e9247bb22b7, regionState=OPENING, regionLocation=9911683f163c,34847,1731435282512 2024-11-12T18:14:46,017 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=96e9ca1050c6b52ea6556fdb4b5d5f39, regionState=OPENING, regionLocation=9911683f163c,34847,1731435282512 2024-11-12T18:14:46,017 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=4aff96e33c9923b5d2448f72171fd310, regionState=OPENING, regionLocation=9911683f163c,34847,1731435282512 2024-11-12T18:14:46,018 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=3a62a194de5221e9f84b2cfd2e83a46a, regionState=OPENING, regionLocation=9911683f163c,34847,1731435282512 2024-11-12T18:14:46,019 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c739eeba765df343af4ef63444655c9f, ASSIGN because future has completed 2024-11-12T18:14:46,019 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure c739eeba765df343af4ef63444655c9f, server=9911683f163c,34847,1731435282512}] 2024-11-12T18:14:46,020 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6a6d3052a2b28af2dab58e9247bb22b7, ASSIGN because future has completed 2024-11-12T18:14:46,021 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=96e9ca1050c6b52ea6556fdb4b5d5f39, ASSIGN because future has completed 2024-11-12T18:14:46,022 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4aff96e33c9923b5d2448f72171fd310, ASSIGN because future has completed 2024-11-12T18:14:46,024 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=3a62a194de5221e9f84b2cfd2e83a46a, ASSIGN because future has completed 2024-11-12T18:14:46,025 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3a62a194de5221e9f84b2cfd2e83a46a, server=9911683f163c,34847,1731435282512}] 2024-11-12T18:14:46,029 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4aff96e33c9923b5d2448f72171fd310, server=9911683f163c,34847,1731435282512}] 2024-11-12T18:14:46,034 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 96e9ca1050c6b52ea6556fdb4b5d5f39, server=9911683f163c,34847,1731435282512}] 2024-11-12T18:14:46,038 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6a6d3052a2b28af2dab58e9247bb22b7, server=9911683f163c,34847,1731435282512}] 2024-11-12T18:14:46,060 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T18:14:46,062 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48161, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T18:14:46,068 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,,1731435284929.5dcfe82d27e00f564e4ad71ea2e78f5c. 2024-11-12T18:14:46,068 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 5dcfe82d27e00f564e4ad71ea2e78f5c, NAME => 'test,,1731435284929.5dcfe82d27e00f564e4ad71ea2e78f5c.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-12T18:14:46,069 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5dcfe82d27e00f564e4ad71ea2e78f5c 2024-11-12T18:14:46,069 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,,1731435284929.5dcfe82d27e00f564e4ad71ea2e78f5c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:46,069 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 5dcfe82d27e00f564e4ad71ea2e78f5c 2024-11-12T18:14:46,069 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 5dcfe82d27e00f564e4ad71ea2e78f5c 2024-11-12T18:14:46,072 INFO [StoreOpener-5dcfe82d27e00f564e4ad71ea2e78f5c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5dcfe82d27e00f564e4ad71ea2e78f5c 2024-11-12T18:14:46,074 INFO [StoreOpener-5dcfe82d27e00f564e4ad71ea2e78f5c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5dcfe82d27e00f564e4ad71ea2e78f5c columnFamilyName family 2024-11-12T18:14:46,074 DEBUG [StoreOpener-5dcfe82d27e00f564e4ad71ea2e78f5c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:46,075 INFO [StoreOpener-5dcfe82d27e00f564e4ad71ea2e78f5c-1 {}] regionserver.HStore(327): Store=5dcfe82d27e00f564e4ad71ea2e78f5c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:46,075 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 5dcfe82d27e00f564e4ad71ea2e78f5c 2024-11-12T18:14:46,076 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/5dcfe82d27e00f564e4ad71ea2e78f5c 2024-11-12T18:14:46,077 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/5dcfe82d27e00f564e4ad71ea2e78f5c 2024-11-12T18:14:46,077 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 5dcfe82d27e00f564e4ad71ea2e78f5c 2024-11-12T18:14:46,077 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 5dcfe82d27e00f564e4ad71ea2e78f5c 2024-11-12T18:14:46,080 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 5dcfe82d27e00f564e4ad71ea2e78f5c 2024-11-12T18:14:46,083 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/5dcfe82d27e00f564e4ad71ea2e78f5c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T18:14:46,084 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 5dcfe82d27e00f564e4ad71ea2e78f5c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74545544, jitterRate=0.11081516742706299}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T18:14:46,084 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5dcfe82d27e00f564e4ad71ea2e78f5c 2024-11-12T18:14:46,086 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 5dcfe82d27e00f564e4ad71ea2e78f5c: Running coprocessor pre-open hook at 1731435286069Writing region info on filesystem at 1731435286069Initializing all the Stores at 1731435286071 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435286071Cleaning up temporary data from old regions at 1731435286078 (+7 ms)Running coprocessor post-open hooks at 1731435286084 (+6 ms)Region opened successfully at 1731435286086 (+2 ms) 2024-11-12T18:14:46,087 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1731435284929.5dcfe82d27e00f564e4ad71ea2e78f5c., pid=34, masterSystemTime=1731435286059 2024-11-12T18:14:46,090 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1731435284929.5dcfe82d27e00f564e4ad71ea2e78f5c. 2024-11-12T18:14:46,090 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,,1731435284929.5dcfe82d27e00f564e4ad71ea2e78f5c. 2024-11-12T18:14:46,090 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,eee,1731435284929.c0b5ce997df05cef2b0f3fd25e49fafc. 2024-11-12T18:14:46,091 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => c0b5ce997df05cef2b0f3fd25e49fafc, NAME => 'test,eee,1731435284929.c0b5ce997df05cef2b0f3fd25e49fafc.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-12T18:14:46,091 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c0b5ce997df05cef2b0f3fd25e49fafc 2024-11-12T18:14:46,091 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5dcfe82d27e00f564e4ad71ea2e78f5c, regionState=OPEN, openSeqNum=2, regionLocation=9911683f163c,45243,1731435282573 2024-11-12T18:14:46,091 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,eee,1731435284929.c0b5ce997df05cef2b0f3fd25e49fafc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:46,091 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for c0b5ce997df05cef2b0f3fd25e49fafc 2024-11-12T18:14:46,091 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for c0b5ce997df05cef2b0f3fd25e49fafc 2024-11-12T18:14:46,094 INFO [StoreOpener-c0b5ce997df05cef2b0f3fd25e49fafc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c0b5ce997df05cef2b0f3fd25e49fafc 2024-11-12T18:14:46,095 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5dcfe82d27e00f564e4ad71ea2e78f5c, server=9911683f163c,45243,1731435282573 because future has completed 2024-11-12T18:14:46,096 INFO [StoreOpener-c0b5ce997df05cef2b0f3fd25e49fafc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0b5ce997df05cef2b0f3fd25e49fafc columnFamilyName family 2024-11-12T18:14:46,096 DEBUG [StoreOpener-c0b5ce997df05cef2b0f3fd25e49fafc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:46,097 INFO [StoreOpener-c0b5ce997df05cef2b0f3fd25e49fafc-1 {}] regionserver.HStore(327): Store=c0b5ce997df05cef2b0f3fd25e49fafc/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:46,098 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for c0b5ce997df05cef2b0f3fd25e49fafc 2024-11-12T18:14:46,099 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/c0b5ce997df05cef2b0f3fd25e49fafc 2024-11-12T18:14:46,099 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/c0b5ce997df05cef2b0f3fd25e49fafc 2024-11-12T18:14:46,100 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=5 2024-11-12T18:14:46,100 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 5dcfe82d27e00f564e4ad71ea2e78f5c, server=9911683f163c,45243,1731435282573 in 172 msec 2024-11-12T18:14:46,100 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for c0b5ce997df05cef2b0f3fd25e49fafc 2024-11-12T18:14:46,100 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for c0b5ce997df05cef2b0f3fd25e49fafc 2024-11-12T18:14:46,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5dcfe82d27e00f564e4ad71ea2e78f5c, ASSIGN in 366 msec 2024-11-12T18:14:46,102 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for c0b5ce997df05cef2b0f3fd25e49fafc 2024-11-12T18:14:46,105 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/c0b5ce997df05cef2b0f3fd25e49fafc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T18:14:46,106 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened c0b5ce997df05cef2b0f3fd25e49fafc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69854039, jitterRate=0.04090629518032074}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T18:14:46,107 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c0b5ce997df05cef2b0f3fd25e49fafc 2024-11-12T18:14:46,107 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for c0b5ce997df05cef2b0f3fd25e49fafc: Running coprocessor pre-open hook at 1731435286092Writing region info on filesystem at 1731435286092Initializing all the Stores at 1731435286093 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435286093Cleaning up temporary data from old regions at 1731435286100 (+7 ms)Running coprocessor post-open hooks at 1731435286107 (+7 ms)Region opened successfully at 1731435286107 2024-11-12T18:14:46,108 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1731435284929.c0b5ce997df05cef2b0f3fd25e49fafc., pid=35, masterSystemTime=1731435286059 2024-11-12T18:14:46,111 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1731435284929.c0b5ce997df05cef2b0f3fd25e49fafc. 2024-11-12T18:14:46,111 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,eee,1731435284929.c0b5ce997df05cef2b0f3fd25e49fafc. 2024-11-12T18:14:46,111 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,ooo,1731435284929.40b076d46dbfff2e1f3c89ad5c3f814d. 2024-11-12T18:14:46,111 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,rrr,1731435284929.1b81a9be48fbe50dde523592a87a4611. 2024-11-12T18:14:46,111 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 1b81a9be48fbe50dde523592a87a4611, NAME => 'test,rrr,1731435284929.1b81a9be48fbe50dde523592a87a4611.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-12T18:14:46,111 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 40b076d46dbfff2e1f3c89ad5c3f814d, NAME => 'test,ooo,1731435284929.40b076d46dbfff2e1f3c89ad5c3f814d.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-12T18:14:46,112 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=c0b5ce997df05cef2b0f3fd25e49fafc, regionState=OPEN, openSeqNum=2, regionLocation=9911683f163c,45243,1731435282573 2024-11-12T18:14:46,112 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 40b076d46dbfff2e1f3c89ad5c3f814d 2024-11-12T18:14:46,112 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1b81a9be48fbe50dde523592a87a4611 2024-11-12T18:14:46,112 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,ooo,1731435284929.40b076d46dbfff2e1f3c89ad5c3f814d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:46,112 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,rrr,1731435284929.1b81a9be48fbe50dde523592a87a4611.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:46,112 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 1b81a9be48fbe50dde523592a87a4611 2024-11-12T18:14:46,112 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 40b076d46dbfff2e1f3c89ad5c3f814d 2024-11-12T18:14:46,112 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 40b076d46dbfff2e1f3c89ad5c3f814d 2024-11-12T18:14:46,112 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 1b81a9be48fbe50dde523592a87a4611 2024-11-12T18:14:46,115 INFO [StoreOpener-1b81a9be48fbe50dde523592a87a4611-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1b81a9be48fbe50dde523592a87a4611 2024-11-12T18:14:46,116 INFO [StoreOpener-40b076d46dbfff2e1f3c89ad5c3f814d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 40b076d46dbfff2e1f3c89ad5c3f814d 2024-11-12T18:14:46,116 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0b5ce997df05cef2b0f3fd25e49fafc, server=9911683f163c,45243,1731435282573 because future has completed 2024-11-12T18:14:46,117 INFO [StoreOpener-1b81a9be48fbe50dde523592a87a4611-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b81a9be48fbe50dde523592a87a4611 columnFamilyName family 2024-11-12T18:14:46,117 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36779 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=9911683f163c,45243,1731435282573, table=test, region=c0b5ce997df05cef2b0f3fd25e49fafc. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-12T18:14:46,117 DEBUG [StoreOpener-1b81a9be48fbe50dde523592a87a4611-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:46,118 INFO [StoreOpener-1b81a9be48fbe50dde523592a87a4611-1 {}] regionserver.HStore(327): Store=1b81a9be48fbe50dde523592a87a4611/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:46,119 INFO [StoreOpener-40b076d46dbfff2e1f3c89ad5c3f814d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40b076d46dbfff2e1f3c89ad5c3f814d columnFamilyName family 2024-11-12T18:14:46,119 DEBUG [StoreOpener-40b076d46dbfff2e1f3c89ad5c3f814d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:46,120 INFO [StoreOpener-40b076d46dbfff2e1f3c89ad5c3f814d-1 {}] regionserver.HStore(327): Store=40b076d46dbfff2e1f3c89ad5c3f814d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:46,120 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 1b81a9be48fbe50dde523592a87a4611 2024-11-12T18:14:46,120 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 40b076d46dbfff2e1f3c89ad5c3f814d 2024-11-12T18:14:46,121 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/1b81a9be48fbe50dde523592a87a4611 2024-11-12T18:14:46,121 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=9 2024-11-12T18:14:46,121 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/40b076d46dbfff2e1f3c89ad5c3f814d 2024-11-12T18:14:46,122 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure c0b5ce997df05cef2b0f3fd25e49fafc, server=9911683f163c,45243,1731435282573 in 189 msec 2024-11-12T18:14:46,122 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/1b81a9be48fbe50dde523592a87a4611 2024-11-12T18:14:46,122 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/40b076d46dbfff2e1f3c89ad5c3f814d 2024-11-12T18:14:46,123 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 1b81a9be48fbe50dde523592a87a4611 2024-11-12T18:14:46,123 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 1b81a9be48fbe50dde523592a87a4611 2024-11-12T18:14:46,123 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 40b076d46dbfff2e1f3c89ad5c3f814d 2024-11-12T18:14:46,123 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 40b076d46dbfff2e1f3c89ad5c3f814d 2024-11-12T18:14:46,125 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c0b5ce997df05cef2b0f3fd25e49fafc, ASSIGN in 388 msec 2024-11-12T18:14:46,126 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 1b81a9be48fbe50dde523592a87a4611 2024-11-12T18:14:46,126 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 40b076d46dbfff2e1f3c89ad5c3f814d 2024-11-12T18:14:46,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T18:14:46,129 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/40b076d46dbfff2e1f3c89ad5c3f814d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T18:14:46,130 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 40b076d46dbfff2e1f3c89ad5c3f814d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64926689, jitterRate=-0.03251694142818451}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T18:14:46,130 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 40b076d46dbfff2e1f3c89ad5c3f814d 2024-11-12T18:14:46,130 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/1b81a9be48fbe50dde523592a87a4611/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T18:14:46,130 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 40b076d46dbfff2e1f3c89ad5c3f814d: Running coprocessor pre-open hook at 1731435286113Writing region info on filesystem at 1731435286113Initializing all the Stores at 1731435286114 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435286114Cleaning up temporary data from old regions at 1731435286123 (+9 ms)Running coprocessor post-open hooks at 1731435286130 (+7 ms)Region opened successfully at 1731435286130 2024-11-12T18:14:46,131 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 1b81a9be48fbe50dde523592a87a4611; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62466163, jitterRate=-0.06918163597583771}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T18:14:46,131 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1b81a9be48fbe50dde523592a87a4611 2024-11-12T18:14:46,131 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 1b81a9be48fbe50dde523592a87a4611: Running coprocessor pre-open hook at 1731435286113Writing region info on filesystem at 1731435286113Initializing all the Stores at 1731435286114 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435286114Cleaning up temporary data from old regions at 1731435286123 (+9 ms)Running coprocessor post-open hooks at 1731435286131 (+8 ms)Region opened successfully at 1731435286131 2024-11-12T18:14:46,131 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1731435284929.40b076d46dbfff2e1f3c89ad5c3f814d., pid=37, masterSystemTime=1731435286059 2024-11-12T18:14:46,132 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1731435284929.1b81a9be48fbe50dde523592a87a4611., pid=44, masterSystemTime=1731435286106 2024-11-12T18:14:46,135 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1731435284929.40b076d46dbfff2e1f3c89ad5c3f814d. 2024-11-12T18:14:46,135 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,ooo,1731435284929.40b076d46dbfff2e1f3c89ad5c3f814d. 2024-11-12T18:14:46,135 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,xxx,1731435284929.0178ff26f4e2d0ab40d50fe6fcd026be. 2024-11-12T18:14:46,135 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 0178ff26f4e2d0ab40d50fe6fcd026be, NAME => 'test,xxx,1731435284929.0178ff26f4e2d0ab40d50fe6fcd026be.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-12T18:14:46,136 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0178ff26f4e2d0ab40d50fe6fcd026be 2024-11-12T18:14:46,136 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=40b076d46dbfff2e1f3c89ad5c3f814d, regionState=OPEN, openSeqNum=2, regionLocation=9911683f163c,45243,1731435282573 2024-11-12T18:14:46,136 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,xxx,1731435284929.0178ff26f4e2d0ab40d50fe6fcd026be.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:46,136 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 0178ff26f4e2d0ab40d50fe6fcd026be 2024-11-12T18:14:46,136 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 0178ff26f4e2d0ab40d50fe6fcd026be 2024-11-12T18:14:46,136 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1731435284929.1b81a9be48fbe50dde523592a87a4611. 2024-11-12T18:14:46,136 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,rrr,1731435284929.1b81a9be48fbe50dde523592a87a4611. 2024-11-12T18:14:46,137 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,vvv,1731435284929.56e168ba1a3103978bee5b6c43feba36. 2024-11-12T18:14:46,137 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => 56e168ba1a3103978bee5b6c43feba36, NAME => 'test,vvv,1731435284929.56e168ba1a3103978bee5b6c43feba36.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-12T18:14:46,137 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 56e168ba1a3103978bee5b6c43feba36 2024-11-12T18:14:46,137 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,vvv,1731435284929.56e168ba1a3103978bee5b6c43feba36.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:46,138 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for 56e168ba1a3103978bee5b6c43feba36 2024-11-12T18:14:46,138 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for 56e168ba1a3103978bee5b6c43feba36 2024-11-12T18:14:46,140 INFO [StoreOpener-0178ff26f4e2d0ab40d50fe6fcd026be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0178ff26f4e2d0ab40d50fe6fcd026be 2024-11-12T18:14:46,141 INFO [StoreOpener-56e168ba1a3103978bee5b6c43feba36-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 56e168ba1a3103978bee5b6c43feba36 2024-11-12T18:14:46,141 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=1b81a9be48fbe50dde523592a87a4611, regionState=OPEN, openSeqNum=2, regionLocation=9911683f163c,40543,1731435282322 2024-11-12T18:14:46,143 INFO [StoreOpener-56e168ba1a3103978bee5b6c43feba36-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56e168ba1a3103978bee5b6c43feba36 columnFamilyName family 2024-11-12T18:14:46,143 DEBUG [StoreOpener-56e168ba1a3103978bee5b6c43feba36-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:46,144 INFO [StoreOpener-56e168ba1a3103978bee5b6c43feba36-1 {}] regionserver.HStore(327): Store=56e168ba1a3103978bee5b6c43feba36/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:46,144 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for 56e168ba1a3103978bee5b6c43feba36 2024-11-12T18:14:46,145 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/56e168ba1a3103978bee5b6c43feba36 2024-11-12T18:14:46,146 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/56e168ba1a3103978bee5b6c43feba36 2024-11-12T18:14:46,146 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for 56e168ba1a3103978bee5b6c43feba36 2024-11-12T18:14:46,146 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for 56e168ba1a3103978bee5b6c43feba36 2024-11-12T18:14:46,147 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T18:14:46,148 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 40b076d46dbfff2e1f3c89ad5c3f814d, server=9911683f163c,45243,1731435282573 because future has completed 2024-11-12T18:14:46,149 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41387, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T18:14:46,150 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for 56e168ba1a3103978bee5b6c43feba36 2024-11-12T18:14:46,150 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b81a9be48fbe50dde523592a87a4611, server=9911683f163c,40543,1731435282322 because future has completed 2024-11-12T18:14:46,151 INFO [StoreOpener-0178ff26f4e2d0ab40d50fe6fcd026be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0178ff26f4e2d0ab40d50fe6fcd026be columnFamilyName family 2024-11-12T18:14:46,151 DEBUG [StoreOpener-0178ff26f4e2d0ab40d50fe6fcd026be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:46,152 INFO [StoreOpener-0178ff26f4e2d0ab40d50fe6fcd026be-1 {}] regionserver.HStore(327): Store=0178ff26f4e2d0ab40d50fe6fcd026be/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:46,152 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 0178ff26f4e2d0ab40d50fe6fcd026be 2024-11-12T18:14:46,154 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/0178ff26f4e2d0ab40d50fe6fcd026be 2024-11-12T18:14:46,155 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=19 2024-11-12T18:14:46,155 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 40b076d46dbfff2e1f3c89ad5c3f814d, server=9911683f163c,45243,1731435282573 in 212 msec 2024-11-12T18:14:46,155 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/0178ff26f4e2d0ab40d50fe6fcd026be 2024-11-12T18:14:46,156 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 0178ff26f4e2d0ab40d50fe6fcd026be 2024-11-12T18:14:46,156 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 0178ff26f4e2d0ab40d50fe6fcd026be 2024-11-12T18:14:46,156 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,mmm,1731435284929.c739eeba765df343af4ef63444655c9f. 2024-11-12T18:14:46,156 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/56e168ba1a3103978bee5b6c43feba36/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T18:14:46,156 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => c739eeba765df343af4ef63444655c9f, NAME => 'test,mmm,1731435284929.c739eeba765df343af4ef63444655c9f.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-12T18:14:46,157 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c739eeba765df343af4ef63444655c9f 2024-11-12T18:14:46,157 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,mmm,1731435284929.c739eeba765df343af4ef63444655c9f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:46,157 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for c739eeba765df343af4ef63444655c9f 2024-11-12T18:14:46,157 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened 56e168ba1a3103978bee5b6c43feba36; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59410152, jitterRate=-0.11471974849700928}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T18:14:46,157 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for c739eeba765df343af4ef63444655c9f 2024-11-12T18:14:46,157 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 56e168ba1a3103978bee5b6c43feba36 2024-11-12T18:14:46,157 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=40b076d46dbfff2e1f3c89ad5c3f814d, ASSIGN in 421 msec 2024-11-12T18:14:46,158 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for 56e168ba1a3103978bee5b6c43feba36: Running coprocessor pre-open hook at 1731435286138Writing region info on filesystem at 1731435286138Initializing all the Stores at 1731435286140 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435286140Cleaning up temporary data from old regions at 1731435286146 (+6 ms)Running coprocessor post-open hooks at 1731435286157 (+11 ms)Region opened successfully at 1731435286158 (+1 ms) 2024-11-12T18:14:46,159 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1731435284929.56e168ba1a3103978bee5b6c43feba36., pid=45, masterSystemTime=1731435286106 2024-11-12T18:14:46,159 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 0178ff26f4e2d0ab40d50fe6fcd026be 2024-11-12T18:14:46,160 INFO [StoreOpener-c739eeba765df343af4ef63444655c9f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c739eeba765df343af4ef63444655c9f 2024-11-12T18:14:46,162 INFO [StoreOpener-c739eeba765df343af4ef63444655c9f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c739eeba765df343af4ef63444655c9f columnFamilyName family 2024-11-12T18:14:46,162 DEBUG [StoreOpener-c739eeba765df343af4ef63444655c9f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:46,163 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1731435284929.56e168ba1a3103978bee5b6c43feba36. 2024-11-12T18:14:46,163 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,vvv,1731435284929.56e168ba1a3103978bee5b6c43feba36. 2024-11-12T18:14:46,163 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,bbb,1731435284929.2248a0d1296d1dbe4f8c9805798db03c. 2024-11-12T18:14:46,163 INFO [StoreOpener-c739eeba765df343af4ef63444655c9f-1 {}] regionserver.HStore(327): Store=c739eeba765df343af4ef63444655c9f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:46,163 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=22 2024-11-12T18:14:46,163 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 2248a0d1296d1dbe4f8c9805798db03c, NAME => 'test,bbb,1731435284929.2248a0d1296d1dbe4f8c9805798db03c.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-12T18:14:46,163 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure 1b81a9be48fbe50dde523592a87a4611, server=9911683f163c,40543,1731435282322 in 183 msec 2024-11-12T18:14:46,163 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for c739eeba765df343af4ef63444655c9f 2024-11-12T18:14:46,164 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2248a0d1296d1dbe4f8c9805798db03c 2024-11-12T18:14:46,164 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,bbb,1731435284929.2248a0d1296d1dbe4f8c9805798db03c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:46,164 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/0178ff26f4e2d0ab40d50fe6fcd026be/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T18:14:46,164 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 2248a0d1296d1dbe4f8c9805798db03c 2024-11-12T18:14:46,164 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 2248a0d1296d1dbe4f8c9805798db03c 2024-11-12T18:14:46,164 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/c739eeba765df343af4ef63444655c9f 2024-11-12T18:14:46,165 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/c739eeba765df343af4ef63444655c9f 2024-11-12T18:14:46,165 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 0178ff26f4e2d0ab40d50fe6fcd026be; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65548665, jitterRate=-0.02324877679347992}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T18:14:46,165 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0178ff26f4e2d0ab40d50fe6fcd026be 2024-11-12T18:14:46,165 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 0178ff26f4e2d0ab40d50fe6fcd026be: Running coprocessor pre-open hook at 1731435286136Writing region info on filesystem at 1731435286136Initializing all the Stores at 1731435286138 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435286138Cleaning up temporary data from old regions at 1731435286156 (+18 ms)Running coprocessor post-open hooks at 1731435286165 (+9 ms)Region opened successfully at 1731435286165 2024-11-12T18:14:46,165 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for c739eeba765df343af4ef63444655c9f 2024-11-12T18:14:46,165 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for c739eeba765df343af4ef63444655c9f 2024-11-12T18:14:46,166 INFO [StoreOpener-2248a0d1296d1dbe4f8c9805798db03c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2248a0d1296d1dbe4f8c9805798db03c 2024-11-12T18:14:46,166 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1731435284929.0178ff26f4e2d0ab40d50fe6fcd026be., pid=31, masterSystemTime=1731435286059 2024-11-12T18:14:46,168 INFO [StoreOpener-2248a0d1296d1dbe4f8c9805798db03c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2248a0d1296d1dbe4f8c9805798db03c columnFamilyName family 2024-11-12T18:14:46,168 DEBUG [StoreOpener-2248a0d1296d1dbe4f8c9805798db03c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:46,169 INFO [StoreOpener-2248a0d1296d1dbe4f8c9805798db03c-1 {}] regionserver.HStore(327): Store=2248a0d1296d1dbe4f8c9805798db03c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:46,169 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN in 429 msec 2024-11-12T18:14:46,169 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 2248a0d1296d1dbe4f8c9805798db03c 2024-11-12T18:14:46,170 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for c739eeba765df343af4ef63444655c9f 2024-11-12T18:14:46,170 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1731435284929.0178ff26f4e2d0ab40d50fe6fcd026be. 2024-11-12T18:14:46,170 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,xxx,1731435284929.0178ff26f4e2d0ab40d50fe6fcd026be. 2024-11-12T18:14:46,170 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/2248a0d1296d1dbe4f8c9805798db03c 2024-11-12T18:14:46,171 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,sss,1731435284929.4795f188f4931ea02df11d7474e4daa4. 2024-11-12T18:14:46,171 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=56e168ba1a3103978bee5b6c43feba36, regionState=OPEN, openSeqNum=2, regionLocation=9911683f163c,40543,1731435282322 2024-11-12T18:14:46,171 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 4795f188f4931ea02df11d7474e4daa4, NAME => 'test,sss,1731435284929.4795f188f4931ea02df11d7474e4daa4.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-12T18:14:46,171 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/2248a0d1296d1dbe4f8c9805798db03c 2024-11-12T18:14:46,171 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4795f188f4931ea02df11d7474e4daa4 2024-11-12T18:14:46,171 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,sss,1731435284929.4795f188f4931ea02df11d7474e4daa4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:46,171 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 4795f188f4931ea02df11d7474e4daa4 2024-11-12T18:14:46,171 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 4795f188f4931ea02df11d7474e4daa4 2024-11-12T18:14:46,172 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 2248a0d1296d1dbe4f8c9805798db03c 2024-11-12T18:14:46,172 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 2248a0d1296d1dbe4f8c9805798db03c 2024-11-12T18:14:46,175 INFO [StoreOpener-4795f188f4931ea02df11d7474e4daa4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4795f188f4931ea02df11d7474e4daa4 2024-11-12T18:14:46,175 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 2248a0d1296d1dbe4f8c9805798db03c 2024-11-12T18:14:46,175 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 56e168ba1a3103978bee5b6c43feba36, server=9911683f163c,40543,1731435282322 because future has completed 2024-11-12T18:14:46,175 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/c739eeba765df343af4ef63444655c9f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T18:14:46,176 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=0178ff26f4e2d0ab40d50fe6fcd026be, regionState=OPEN, openSeqNum=2, regionLocation=9911683f163c,45243,1731435282573 2024-11-12T18:14:46,176 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened c739eeba765df343af4ef63444655c9f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60943514, jitterRate=-0.09187087416648865}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T18:14:46,176 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c739eeba765df343af4ef63444655c9f 2024-11-12T18:14:46,177 INFO [StoreOpener-4795f188f4931ea02df11d7474e4daa4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4795f188f4931ea02df11d7474e4daa4 columnFamilyName family 2024-11-12T18:14:46,177 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for c739eeba765df343af4ef63444655c9f: Running coprocessor pre-open hook at 1731435286158Writing region info on filesystem at 1731435286158Initializing all the Stores at 1731435286159 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435286159Cleaning up temporary data from old regions at 1731435286166 (+7 ms)Running coprocessor post-open hooks at 1731435286177 (+11 ms)Region opened successfully at 1731435286177 2024-11-12T18:14:46,177 DEBUG [StoreOpener-4795f188f4931ea02df11d7474e4daa4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:46,178 INFO [StoreOpener-4795f188f4931ea02df11d7474e4daa4-1 {}] regionserver.HStore(327): Store=4795f188f4931ea02df11d7474e4daa4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:46,178 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 4795f188f4931ea02df11d7474e4daa4 2024-11-12T18:14:46,179 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/4795f188f4931ea02df11d7474e4daa4 2024-11-12T18:14:46,181 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0178ff26f4e2d0ab40d50fe6fcd026be, server=9911683f163c,45243,1731435282573 because future has completed 2024-11-12T18:14:46,181 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/4795f188f4931ea02df11d7474e4daa4 2024-11-12T18:14:46,181 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1731435284929.c739eeba765df343af4ef63444655c9f., pid=52, masterSystemTime=1731435286147 2024-11-12T18:14:46,182 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 4795f188f4931ea02df11d7474e4daa4 2024-11-12T18:14:46,182 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 4795f188f4931ea02df11d7474e4daa4 2024-11-12T18:14:46,184 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/2248a0d1296d1dbe4f8c9805798db03c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T18:14:46,185 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 4795f188f4931ea02df11d7474e4daa4 2024-11-12T18:14:46,185 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1731435284929.c739eeba765df343af4ef63444655c9f. 2024-11-12T18:14:46,185 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,mmm,1731435284929.c739eeba765df343af4ef63444655c9f. 2024-11-12T18:14:46,186 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,iii,1731435284929.96e9ca1050c6b52ea6556fdb4b5d5f39. 2024-11-12T18:14:46,187 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 2248a0d1296d1dbe4f8c9805798db03c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61962966, jitterRate=-0.07667985558509827}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T18:14:46,187 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2248a0d1296d1dbe4f8c9805798db03c 2024-11-12T18:14:46,187 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=26 2024-11-12T18:14:46,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 56e168ba1a3103978bee5b6c43feba36, server=9911683f163c,40543,1731435282322 in 202 msec 2024-11-12T18:14:46,188 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 2248a0d1296d1dbe4f8c9805798db03c: Running coprocessor pre-open hook at 1731435286164Writing region info on filesystem at 1731435286164Initializing all the Stores at 1731435286165 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435286165Cleaning up temporary data from old regions at 1731435286172 (+7 ms)Running coprocessor post-open hooks at 1731435286187 (+15 ms)Region opened successfully at 1731435286188 (+1 ms) 2024-11-12T18:14:46,188 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 96e9ca1050c6b52ea6556fdb4b5d5f39, NAME => 'test,iii,1731435284929.96e9ca1050c6b52ea6556fdb4b5d5f39.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-12T18:14:46,188 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 96e9ca1050c6b52ea6556fdb4b5d5f39 2024-11-12T18:14:46,188 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,iii,1731435284929.96e9ca1050c6b52ea6556fdb4b5d5f39.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:46,189 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 96e9ca1050c6b52ea6556fdb4b5d5f39 2024-11-12T18:14:46,189 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 96e9ca1050c6b52ea6556fdb4b5d5f39 2024-11-12T18:14:46,189 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1731435284929.2248a0d1296d1dbe4f8c9805798db03c., pid=51, masterSystemTime=1731435286106 2024-11-12T18:14:46,190 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/4795f188f4931ea02df11d7474e4daa4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T18:14:46,191 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 4795f188f4931ea02df11d7474e4daa4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64610461, jitterRate=-0.037229105830192566}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T18:14:46,191 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4795f188f4931ea02df11d7474e4daa4 2024-11-12T18:14:46,191 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 4795f188f4931ea02df11d7474e4daa4: Running coprocessor pre-open hook at 1731435286172Writing region info on filesystem at 1731435286172Initializing all the Stores at 1731435286173 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435286173Cleaning up temporary data from old regions at 1731435286182 (+9 ms)Running coprocessor post-open hooks at 1731435286191 (+9 ms)Region opened successfully at 1731435286191 2024-11-12T18:14:46,192 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN in 454 msec 2024-11-12T18:14:46,193 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1731435284929.4795f188f4931ea02df11d7474e4daa4., pid=38, masterSystemTime=1731435286059 2024-11-12T18:14:46,194 INFO [StoreOpener-96e9ca1050c6b52ea6556fdb4b5d5f39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 96e9ca1050c6b52ea6556fdb4b5d5f39 2024-11-12T18:14:46,194 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1731435284929.2248a0d1296d1dbe4f8c9805798db03c. 2024-11-12T18:14:46,195 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,bbb,1731435284929.2248a0d1296d1dbe4f8c9805798db03c. 2024-11-12T18:14:46,195 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=c739eeba765df343af4ef63444655c9f, regionState=OPEN, openSeqNum=2, regionLocation=9911683f163c,34847,1731435282512 2024-11-12T18:14:46,195 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,yyy,1731435284929.fde2b7fbe9fa7c8a87ab544f5542d82a. 2024-11-12T18:14:46,195 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => fde2b7fbe9fa7c8a87ab544f5542d82a, NAME => 'test,yyy,1731435284929.fde2b7fbe9fa7c8a87ab544f5542d82a.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-12T18:14:46,195 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test fde2b7fbe9fa7c8a87ab544f5542d82a 2024-11-12T18:14:46,196 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,yyy,1731435284929.fde2b7fbe9fa7c8a87ab544f5542d82a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:46,196 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for fde2b7fbe9fa7c8a87ab544f5542d82a 2024-11-12T18:14:46,196 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for fde2b7fbe9fa7c8a87ab544f5542d82a 2024-11-12T18:14:46,198 INFO [StoreOpener-96e9ca1050c6b52ea6556fdb4b5d5f39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 96e9ca1050c6b52ea6556fdb4b5d5f39 columnFamilyName family 2024-11-12T18:14:46,199 DEBUG [StoreOpener-96e9ca1050c6b52ea6556fdb4b5d5f39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:46,200 INFO [StoreOpener-96e9ca1050c6b52ea6556fdb4b5d5f39-1 {}] regionserver.HStore(327): Store=96e9ca1050c6b52ea6556fdb4b5d5f39/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:46,200 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 96e9ca1050c6b52ea6556fdb4b5d5f39 2024-11-12T18:14:46,200 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure c739eeba765df343af4ef63444655c9f, server=9911683f163c,34847,1731435282512 because future has completed 2024-11-12T18:14:46,201 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1731435284929.4795f188f4931ea02df11d7474e4daa4. 2024-11-12T18:14:46,201 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,sss,1731435284929.4795f188f4931ea02df11d7474e4daa4. 2024-11-12T18:14:46,201 INFO [StoreOpener-fde2b7fbe9fa7c8a87ab544f5542d82a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region fde2b7fbe9fa7c8a87ab544f5542d82a 2024-11-12T18:14:46,201 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,hhh,1731435284929.dff0a9c660bd5084b34753ea40ea3278. 2024-11-12T18:14:46,201 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/96e9ca1050c6b52ea6556fdb4b5d5f39 2024-11-12T18:14:46,201 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => dff0a9c660bd5084b34753ea40ea3278, NAME => 'test,hhh,1731435284929.dff0a9c660bd5084b34753ea40ea3278.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-12T18:14:46,201 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test dff0a9c660bd5084b34753ea40ea3278 2024-11-12T18:14:46,201 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,hhh,1731435284929.dff0a9c660bd5084b34753ea40ea3278.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:46,202 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/96e9ca1050c6b52ea6556fdb4b5d5f39 2024-11-12T18:14:46,202 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for dff0a9c660bd5084b34753ea40ea3278 2024-11-12T18:14:46,202 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for dff0a9c660bd5084b34753ea40ea3278 2024-11-12T18:14:46,202 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 96e9ca1050c6b52ea6556fdb4b5d5f39 2024-11-12T18:14:46,202 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 96e9ca1050c6b52ea6556fdb4b5d5f39 2024-11-12T18:14:46,203 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=4795f188f4931ea02df11d7474e4daa4, regionState=OPEN, openSeqNum=2, regionLocation=9911683f163c,45243,1731435282573 2024-11-12T18:14:46,204 INFO [StoreOpener-fde2b7fbe9fa7c8a87ab544f5542d82a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fde2b7fbe9fa7c8a87ab544f5542d82a columnFamilyName family 2024-11-12T18:14:46,204 DEBUG [StoreOpener-fde2b7fbe9fa7c8a87ab544f5542d82a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:46,204 INFO [StoreOpener-dff0a9c660bd5084b34753ea40ea3278-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region dff0a9c660bd5084b34753ea40ea3278 2024-11-12T18:14:46,205 INFO [StoreOpener-fde2b7fbe9fa7c8a87ab544f5542d82a-1 {}] regionserver.HStore(327): Store=fde2b7fbe9fa7c8a87ab544f5542d82a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:46,205 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for fde2b7fbe9fa7c8a87ab544f5542d82a 2024-11-12T18:14:46,206 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 96e9ca1050c6b52ea6556fdb4b5d5f39 2024-11-12T18:14:46,207 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4795f188f4931ea02df11d7474e4daa4, server=9911683f163c,45243,1731435282573 because future has completed 2024-11-12T18:14:46,210 INFO [StoreOpener-dff0a9c660bd5084b34753ea40ea3278-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dff0a9c660bd5084b34753ea40ea3278 columnFamilyName family 2024-11-12T18:14:46,210 DEBUG [StoreOpener-dff0a9c660bd5084b34753ea40ea3278-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:46,211 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=17 2024-11-12T18:14:46,211 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure c739eeba765df343af4ef63444655c9f, server=9911683f163c,34847,1731435282512 in 188 msec 2024-11-12T18:14:46,213 INFO [StoreOpener-dff0a9c660bd5084b34753ea40ea3278-1 {}] regionserver.HStore(327): Store=dff0a9c660bd5084b34753ea40ea3278/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:46,213 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for dff0a9c660bd5084b34753ea40ea3278 2024-11-12T18:14:46,214 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/fde2b7fbe9fa7c8a87ab544f5542d82a 2024-11-12T18:14:46,214 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/fde2b7fbe9fa7c8a87ab544f5542d82a 2024-11-12T18:14:46,214 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/dff0a9c660bd5084b34753ea40ea3278 2024-11-12T18:14:46,215 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/dff0a9c660bd5084b34753ea40ea3278 2024-11-12T18:14:46,215 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for fde2b7fbe9fa7c8a87ab544f5542d82a 2024-11-12T18:14:46,215 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for fde2b7fbe9fa7c8a87ab544f5542d82a 2024-11-12T18:14:46,216 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for dff0a9c660bd5084b34753ea40ea3278 2024-11-12T18:14:46,216 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for dff0a9c660bd5084b34753ea40ea3278 2024-11-12T18:14:46,216 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/96e9ca1050c6b52ea6556fdb4b5d5f39/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T18:14:46,217 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c739eeba765df343af4ef63444655c9f, ASSIGN in 477 msec 2024-11-12T18:14:46,219 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for dff0a9c660bd5084b34753ea40ea3278 2024-11-12T18:14:46,219 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 96e9ca1050c6b52ea6556fdb4b5d5f39; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66267055, jitterRate=-0.012543931603431702}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T18:14:46,220 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 96e9ca1050c6b52ea6556fdb4b5d5f39 2024-11-12T18:14:46,220 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 96e9ca1050c6b52ea6556fdb4b5d5f39: Running coprocessor pre-open hook at 1731435286189Writing region info on filesystem at 1731435286189Initializing all the Stores at 1731435286191 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435286191Cleaning up temporary data from old regions at 1731435286202 (+11 ms)Running coprocessor post-open hooks at 1731435286220 (+18 ms)Region opened successfully at 1731435286220 2024-11-12T18:14:46,221 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1731435284929.96e9ca1050c6b52ea6556fdb4b5d5f39., pid=55, masterSystemTime=1731435286147 2024-11-12T18:14:46,225 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for fde2b7fbe9fa7c8a87ab544f5542d82a 2024-11-12T18:14:46,228 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/dff0a9c660bd5084b34753ea40ea3278/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T18:14:46,228 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened dff0a9c660bd5084b34753ea40ea3278; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66146735, jitterRate=-0.014336839318275452}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T18:14:46,229 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dff0a9c660bd5084b34753ea40ea3278 2024-11-12T18:14:46,229 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for dff0a9c660bd5084b34753ea40ea3278: Running coprocessor pre-open hook at 1731435286202Writing region info on filesystem at 1731435286202Initializing all the Stores at 1731435286203 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435286203Cleaning up temporary data from old regions at 1731435286216 (+13 ms)Running coprocessor post-open hooks at 1731435286229 (+13 ms)Region opened successfully at 1731435286229 2024-11-12T18:14:46,230 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1731435284929.dff0a9c660bd5084b34753ea40ea3278., pid=33, masterSystemTime=1731435286059 2024-11-12T18:14:46,231 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=23 2024-11-12T18:14:46,231 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 4795f188f4931ea02df11d7474e4daa4, server=9911683f163c,45243,1731435282573 in 283 msec 2024-11-12T18:14:46,234 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4795f188f4931ea02df11d7474e4daa4, ASSIGN in 498 msec 2024-11-12T18:14:46,236 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=96e9ca1050c6b52ea6556fdb4b5d5f39, regionState=OPEN, openSeqNum=2, regionLocation=9911683f163c,34847,1731435282512 2024-11-12T18:14:46,240 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 96e9ca1050c6b52ea6556fdb4b5d5f39, server=9911683f163c,34847,1731435282512 because future has completed 2024-11-12T18:14:46,240 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/fde2b7fbe9fa7c8a87ab544f5542d82a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T18:14:46,240 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1731435284929.96e9ca1050c6b52ea6556fdb4b5d5f39. 2024-11-12T18:14:46,241 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,iii,1731435284929.96e9ca1050c6b52ea6556fdb4b5d5f39. 2024-11-12T18:14:46,241 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,qqq,1731435284929.6d7c21378c2e7788ac010db26909da83. 2024-11-12T18:14:46,241 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => 6d7c21378c2e7788ac010db26909da83, NAME => 'test,qqq,1731435284929.6d7c21378c2e7788ac010db26909da83.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-12T18:14:46,241 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1731435284929.dff0a9c660bd5084b34753ea40ea3278. 2024-11-12T18:14:46,241 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,hhh,1731435284929.dff0a9c660bd5084b34753ea40ea3278. 2024-11-12T18:14:46,241 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,uuu,1731435284929.a7b0558136890d45b56827e5af943cfa. 2024-11-12T18:14:46,242 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36779 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=9911683f163c,45243,1731435282573, table=test, region=0178ff26f4e2d0ab40d50fe6fcd026be. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-12T18:14:46,242 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6d7c21378c2e7788ac010db26909da83 2024-11-12T18:14:46,242 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => a7b0558136890d45b56827e5af943cfa, NAME => 'test,uuu,1731435284929.a7b0558136890d45b56827e5af943cfa.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-12T18:14:46,243 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,qqq,1731435284929.6d7c21378c2e7788ac010db26909da83.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:46,243 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for 6d7c21378c2e7788ac010db26909da83 2024-11-12T18:14:46,243 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for 6d7c21378c2e7788ac010db26909da83 2024-11-12T18:14:46,243 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a7b0558136890d45b56827e5af943cfa 2024-11-12T18:14:46,243 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,uuu,1731435284929.a7b0558136890d45b56827e5af943cfa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:46,243 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for a7b0558136890d45b56827e5af943cfa 2024-11-12T18:14:46,243 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for a7b0558136890d45b56827e5af943cfa 2024-11-12T18:14:46,245 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened fde2b7fbe9fa7c8a87ab544f5542d82a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66243940, jitterRate=-0.01288837194442749}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T18:14:46,245 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fde2b7fbe9fa7c8a87ab544f5542d82a 2024-11-12T18:14:46,245 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for fde2b7fbe9fa7c8a87ab544f5542d82a: Running coprocessor pre-open hook at 1731435286196Writing region info on filesystem at 1731435286196Initializing all the Stores at 1731435286200 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435286200Cleaning up temporary data from old regions at 1731435286215 (+15 ms)Running coprocessor post-open hooks at 1731435286245 (+30 ms)Region opened successfully at 1731435286245 2024-11-12T18:14:46,246 INFO [StoreOpener-6d7c21378c2e7788ac010db26909da83-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6d7c21378c2e7788ac010db26909da83 2024-11-12T18:14:46,246 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1731435284929.fde2b7fbe9fa7c8a87ab544f5542d82a., pid=39, masterSystemTime=1731435286106 2024-11-12T18:14:46,247 INFO [StoreOpener-a7b0558136890d45b56827e5af943cfa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a7b0558136890d45b56827e5af943cfa 2024-11-12T18:14:46,248 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=13 2024-11-12T18:14:46,248 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure 96e9ca1050c6b52ea6556fdb4b5d5f39, server=9911683f163c,34847,1731435282512 in 210 msec 2024-11-12T18:14:46,248 INFO [StoreOpener-6d7c21378c2e7788ac010db26909da83-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6d7c21378c2e7788ac010db26909da83 columnFamilyName family 2024-11-12T18:14:46,248 DEBUG [StoreOpener-6d7c21378c2e7788ac010db26909da83-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:46,249 INFO [StoreOpener-6d7c21378c2e7788ac010db26909da83-1 {}] regionserver.HStore(327): Store=6d7c21378c2e7788ac010db26909da83/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:46,250 INFO [StoreOpener-a7b0558136890d45b56827e5af943cfa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a7b0558136890d45b56827e5af943cfa columnFamilyName family 2024-11-12T18:14:46,250 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for 6d7c21378c2e7788ac010db26909da83 2024-11-12T18:14:46,250 DEBUG [StoreOpener-a7b0558136890d45b56827e5af943cfa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:46,250 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1731435284929.fde2b7fbe9fa7c8a87ab544f5542d82a. 2024-11-12T18:14:46,250 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,yyy,1731435284929.fde2b7fbe9fa7c8a87ab544f5542d82a. 2024-11-12T18:14:46,250 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,nnn,1731435284929.4b703e982cd6bbf72ec2f771b120c649. 2024-11-12T18:14:46,250 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 4b703e982cd6bbf72ec2f771b120c649, NAME => 'test,nnn,1731435284929.4b703e982cd6bbf72ec2f771b120c649.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-12T18:14:46,250 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4b703e982cd6bbf72ec2f771b120c649 2024-11-12T18:14:46,251 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,nnn,1731435284929.4b703e982cd6bbf72ec2f771b120c649.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:46,251 INFO [StoreOpener-a7b0558136890d45b56827e5af943cfa-1 {}] regionserver.HStore(327): Store=a7b0558136890d45b56827e5af943cfa/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:46,251 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 4b703e982cd6bbf72ec2f771b120c649 2024-11-12T18:14:46,251 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 4b703e982cd6bbf72ec2f771b120c649 2024-11-12T18:14:46,251 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for a7b0558136890d45b56827e5af943cfa 2024-11-12T18:14:46,251 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/6d7c21378c2e7788ac010db26909da83 2024-11-12T18:14:46,252 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/6d7c21378c2e7788ac010db26909da83 2024-11-12T18:14:46,252 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for 6d7c21378c2e7788ac010db26909da83 2024-11-12T18:14:46,253 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for 6d7c21378c2e7788ac010db26909da83 2024-11-12T18:14:46,255 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/a7b0558136890d45b56827e5af943cfa 2024-11-12T18:14:46,256 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for 6d7c21378c2e7788ac010db26909da83 2024-11-12T18:14:46,256 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/a7b0558136890d45b56827e5af943cfa 2024-11-12T18:14:46,257 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for a7b0558136890d45b56827e5af943cfa 2024-11-12T18:14:46,257 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for a7b0558136890d45b56827e5af943cfa 2024-11-12T18:14:46,260 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/6d7c21378c2e7788ac010db26909da83/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T18:14:46,260 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for a7b0558136890d45b56827e5af943cfa 2024-11-12T18:14:46,261 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened 6d7c21378c2e7788ac010db26909da83; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75220442, jitterRate=0.12087193131446838}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T18:14:46,261 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6d7c21378c2e7788ac010db26909da83 2024-11-12T18:14:46,261 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for 6d7c21378c2e7788ac010db26909da83: Running coprocessor pre-open hook at 1731435286243Writing region info on filesystem at 1731435286243Initializing all the Stores at 1731435286245 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435286245Cleaning up temporary data from old regions at 1731435286253 (+8 ms)Running coprocessor post-open hooks at 1731435286261 (+8 ms)Region opened successfully at 1731435286261 2024-11-12T18:14:46,261 INFO [StoreOpener-4b703e982cd6bbf72ec2f771b120c649-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4b703e982cd6bbf72ec2f771b120c649 2024-11-12T18:14:46,262 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1731435284929.6d7c21378c2e7788ac010db26909da83., pid=48, masterSystemTime=1731435286147 2024-11-12T18:14:46,263 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=96e9ca1050c6b52ea6556fdb4b5d5f39, ASSIGN in 514 msec 2024-11-12T18:14:46,265 INFO [StoreOpener-4b703e982cd6bbf72ec2f771b120c649-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b703e982cd6bbf72ec2f771b120c649 columnFamilyName family 2024-11-12T18:14:46,265 DEBUG [StoreOpener-4b703e982cd6bbf72ec2f771b120c649-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:46,266 INFO [StoreOpener-4b703e982cd6bbf72ec2f771b120c649-1 {}] regionserver.HStore(327): Store=4b703e982cd6bbf72ec2f771b120c649/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:46,266 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 4b703e982cd6bbf72ec2f771b120c649 2024-11-12T18:14:46,267 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/4b703e982cd6bbf72ec2f771b120c649 2024-11-12T18:14:46,268 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/4b703e982cd6bbf72ec2f771b120c649 2024-11-12T18:14:46,269 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 4b703e982cd6bbf72ec2f771b120c649 2024-11-12T18:14:46,269 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 4b703e982cd6bbf72ec2f771b120c649 2024-11-12T18:14:46,271 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=fde2b7fbe9fa7c8a87ab544f5542d82a, regionState=OPEN, openSeqNum=2, regionLocation=9911683f163c,40543,1731435282322 2024-11-12T18:14:46,271 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 4b703e982cd6bbf72ec2f771b120c649 2024-11-12T18:14:46,272 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1731435284929.6d7c21378c2e7788ac010db26909da83. 2024-11-12T18:14:46,272 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,qqq,1731435284929.6d7c21378c2e7788ac010db26909da83. 2024-11-12T18:14:46,272 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,jjj,1731435284929.6a6d3052a2b28af2dab58e9247bb22b7. 2024-11-12T18:14:46,272 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => 6a6d3052a2b28af2dab58e9247bb22b7, NAME => 'test,jjj,1731435284929.6a6d3052a2b28af2dab58e9247bb22b7.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-12T18:14:46,273 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6a6d3052a2b28af2dab58e9247bb22b7 2024-11-12T18:14:46,273 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,jjj,1731435284929.6a6d3052a2b28af2dab58e9247bb22b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:46,273 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for 6a6d3052a2b28af2dab58e9247bb22b7 2024-11-12T18:14:46,273 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for 6a6d3052a2b28af2dab58e9247bb22b7 2024-11-12T18:14:46,275 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/a7b0558136890d45b56827e5af943cfa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T18:14:46,278 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure fde2b7fbe9fa7c8a87ab544f5542d82a, server=9911683f163c,40543,1731435282322 because future has completed 2024-11-12T18:14:46,279 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened a7b0558136890d45b56827e5af943cfa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61454971, jitterRate=-0.0842495709657669}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T18:14:46,279 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a7b0558136890d45b56827e5af943cfa 2024-11-12T18:14:46,279 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for a7b0558136890d45b56827e5af943cfa: Running coprocessor pre-open hook at 1731435286243Writing region info on filesystem at 1731435286243Initializing all the Stores at 1731435286246 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435286246Cleaning up temporary data from old regions at 1731435286257 (+11 ms)Running coprocessor post-open hooks at 1731435286279 (+22 ms)Region opened successfully at 1731435286279 2024-11-12T18:14:46,280 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1731435284929.a7b0558136890d45b56827e5af943cfa., pid=32, masterSystemTime=1731435286059 2024-11-12T18:14:46,280 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=6d7c21378c2e7788ac010db26909da83, regionState=OPEN, openSeqNum=2, regionLocation=9911683f163c,34847,1731435282512 2024-11-12T18:14:46,285 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d7c21378c2e7788ac010db26909da83, server=9911683f163c,34847,1731435282512 because future has completed 2024-11-12T18:14:46,286 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1731435284929.a7b0558136890d45b56827e5af943cfa. 2024-11-12T18:14:46,287 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,uuu,1731435284929.a7b0558136890d45b56827e5af943cfa. 2024-11-12T18:14:46,287 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,lll,1731435284929.394b0963ad0828345e14c3be23cb7790. 2024-11-12T18:14:46,287 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 394b0963ad0828345e14c3be23cb7790, NAME => 'test,lll,1731435284929.394b0963ad0828345e14c3be23cb7790.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-12T18:14:46,288 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 394b0963ad0828345e14c3be23cb7790 2024-11-12T18:14:46,288 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,lll,1731435284929.394b0963ad0828345e14c3be23cb7790.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:46,288 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 394b0963ad0828345e14c3be23cb7790 2024-11-12T18:14:46,288 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 394b0963ad0828345e14c3be23cb7790 2024-11-12T18:14:46,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=29 2024-11-12T18:14:46,291 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/4b703e982cd6bbf72ec2f771b120c649/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T18:14:46,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure fde2b7fbe9fa7c8a87ab544f5542d82a, server=9911683f163c,40543,1731435282322 in 334 msec 2024-11-12T18:14:46,293 INFO [StoreOpener-6a6d3052a2b28af2dab58e9247bb22b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6a6d3052a2b28af2dab58e9247bb22b7 2024-11-12T18:14:46,293 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 4b703e982cd6bbf72ec2f771b120c649; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60830937, jitterRate=-0.09354840219020844}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T18:14:46,293 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4b703e982cd6bbf72ec2f771b120c649 2024-11-12T18:14:46,294 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN in 557 msec 2024-11-12T18:14:46,294 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 4b703e982cd6bbf72ec2f771b120c649: Running coprocessor pre-open hook at 1731435286251Writing region info on filesystem at 1731435286251Initializing all the Stores at 1731435286252 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435286253 (+1 ms)Cleaning up temporary data from old regions at 1731435286269 (+16 ms)Running coprocessor post-open hooks at 1731435286293 (+24 ms)Region opened successfully at 1731435286293 2024-11-12T18:14:46,295 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1731435284929.4b703e982cd6bbf72ec2f771b120c649., pid=40, masterSystemTime=1731435286106 2024-11-12T18:14:46,295 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=a7b0558136890d45b56827e5af943cfa, regionState=OPEN, openSeqNum=2, regionLocation=9911683f163c,45243,1731435282573 2024-11-12T18:14:46,298 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1731435284929.4b703e982cd6bbf72ec2f771b120c649. 2024-11-12T18:14:46,299 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,nnn,1731435284929.4b703e982cd6bbf72ec2f771b120c649. 2024-11-12T18:14:46,299 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,ppp,1731435284929.9fbe24cda440bbf727cd581ab33ad7a9. 2024-11-12T18:14:46,299 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 9fbe24cda440bbf727cd581ab33ad7a9, NAME => 'test,ppp,1731435284929.9fbe24cda440bbf727cd581ab33ad7a9.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-12T18:14:46,299 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7b0558136890d45b56827e5af943cfa, server=9911683f163c,45243,1731435282573 because future has completed 2024-11-12T18:14:46,300 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 9fbe24cda440bbf727cd581ab33ad7a9 2024-11-12T18:14:46,300 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,ppp,1731435284929.9fbe24cda440bbf727cd581ab33ad7a9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:46,300 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 9fbe24cda440bbf727cd581ab33ad7a9 2024-11-12T18:14:46,300 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 9fbe24cda440bbf727cd581ab33ad7a9 2024-11-12T18:14:46,304 INFO [StoreOpener-6a6d3052a2b28af2dab58e9247bb22b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a6d3052a2b28af2dab58e9247bb22b7 columnFamilyName family 2024-11-12T18:14:46,304 DEBUG [StoreOpener-6a6d3052a2b28af2dab58e9247bb22b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:46,305 INFO [StoreOpener-394b0963ad0828345e14c3be23cb7790-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 394b0963ad0828345e14c3be23cb7790 2024-11-12T18:14:46,305 INFO [StoreOpener-6a6d3052a2b28af2dab58e9247bb22b7-1 {}] regionserver.HStore(327): Store=6a6d3052a2b28af2dab58e9247bb22b7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:46,306 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for 6a6d3052a2b28af2dab58e9247bb22b7 2024-11-12T18:14:46,306 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=21 2024-11-12T18:14:46,306 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 6d7c21378c2e7788ac010db26909da83, server=9911683f163c,34847,1731435282512 in 302 msec 2024-11-12T18:14:46,306 INFO [StoreOpener-9fbe24cda440bbf727cd581ab33ad7a9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 9fbe24cda440bbf727cd581ab33ad7a9 2024-11-12T18:14:46,307 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/6a6d3052a2b28af2dab58e9247bb22b7 2024-11-12T18:14:46,308 INFO [StoreOpener-394b0963ad0828345e14c3be23cb7790-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 394b0963ad0828345e14c3be23cb7790 columnFamilyName family 2024-11-12T18:14:46,308 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/6a6d3052a2b28af2dab58e9247bb22b7 2024-11-12T18:14:46,308 DEBUG [StoreOpener-394b0963ad0828345e14c3be23cb7790-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:46,309 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6d7c21378c2e7788ac010db26909da83, ASSIGN in 572 msec 2024-11-12T18:14:46,310 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for 6a6d3052a2b28af2dab58e9247bb22b7 2024-11-12T18:14:46,310 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for 6a6d3052a2b28af2dab58e9247bb22b7 2024-11-12T18:14:46,310 INFO [StoreOpener-394b0963ad0828345e14c3be23cb7790-1 {}] regionserver.HStore(327): Store=394b0963ad0828345e14c3be23cb7790/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:46,310 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 394b0963ad0828345e14c3be23cb7790 2024-11-12T18:14:46,311 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/394b0963ad0828345e14c3be23cb7790 2024-11-12T18:14:46,312 INFO [StoreOpener-9fbe24cda440bbf727cd581ab33ad7a9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9fbe24cda440bbf727cd581ab33ad7a9 columnFamilyName family 2024-11-12T18:14:46,312 DEBUG [StoreOpener-9fbe24cda440bbf727cd581ab33ad7a9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:46,312 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/394b0963ad0828345e14c3be23cb7790 2024-11-12T18:14:46,313 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 394b0963ad0828345e14c3be23cb7790 2024-11-12T18:14:46,313 INFO [StoreOpener-9fbe24cda440bbf727cd581ab33ad7a9-1 {}] regionserver.HStore(327): Store=9fbe24cda440bbf727cd581ab33ad7a9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:46,313 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 394b0963ad0828345e14c3be23cb7790 2024-11-12T18:14:46,313 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 9fbe24cda440bbf727cd581ab33ad7a9 2024-11-12T18:14:46,314 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for 6a6d3052a2b28af2dab58e9247bb22b7 2024-11-12T18:14:46,314 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/9fbe24cda440bbf727cd581ab33ad7a9 2024-11-12T18:14:46,316 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/9fbe24cda440bbf727cd581ab33ad7a9 2024-11-12T18:14:46,317 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 394b0963ad0828345e14c3be23cb7790 2024-11-12T18:14:46,318 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 9fbe24cda440bbf727cd581ab33ad7a9 2024-11-12T18:14:46,318 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 9fbe24cda440bbf727cd581ab33ad7a9 2024-11-12T18:14:46,322 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 9fbe24cda440bbf727cd581ab33ad7a9 2024-11-12T18:14:46,324 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/6a6d3052a2b28af2dab58e9247bb22b7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T18:14:46,325 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened 6a6d3052a2b28af2dab58e9247bb22b7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73215386, jitterRate=0.09099426865577698}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T18:14:46,325 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6a6d3052a2b28af2dab58e9247bb22b7 2024-11-12T18:14:46,326 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for 6a6d3052a2b28af2dab58e9247bb22b7: Running coprocessor pre-open hook at 1731435286273Writing region info on filesystem at 1731435286273Initializing all the Stores at 1731435286278 (+5 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435286278Cleaning up temporary data from old regions at 1731435286310 (+32 ms)Running coprocessor post-open hooks at 1731435286325 (+15 ms)Region opened successfully at 1731435286325 2024-11-12T18:14:46,327 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1731435284929.6a6d3052a2b28af2dab58e9247bb22b7., pid=56, masterSystemTime=1731435286147 2024-11-12T18:14:46,328 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/394b0963ad0828345e14c3be23cb7790/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T18:14:46,329 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 394b0963ad0828345e14c3be23cb7790; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59892718, jitterRate=-0.10752895474433899}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T18:14:46,329 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 394b0963ad0828345e14c3be23cb7790 2024-11-12T18:14:46,330 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 394b0963ad0828345e14c3be23cb7790: Running coprocessor pre-open hook at 1731435286288Writing region info on filesystem at 1731435286288Initializing all the Stores at 1731435286298 (+10 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435286298Cleaning up temporary data from old regions at 1731435286313 (+15 ms)Running coprocessor post-open hooks at 1731435286330 (+17 ms)Region opened successfully at 1731435286330 2024-11-12T18:14:46,330 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/9fbe24cda440bbf727cd581ab33ad7a9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T18:14:46,331 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=25 2024-11-12T18:14:46,331 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure a7b0558136890d45b56827e5af943cfa, server=9911683f163c,45243,1731435282573 in 403 msec 2024-11-12T18:14:46,333 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 9fbe24cda440bbf727cd581ab33ad7a9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66285967, jitterRate=-0.012262120842933655}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T18:14:46,333 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9fbe24cda440bbf727cd581ab33ad7a9 2024-11-12T18:14:46,333 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 9fbe24cda440bbf727cd581ab33ad7a9: Running coprocessor pre-open hook at 1731435286300Writing region info on filesystem at 1731435286300Initializing all the Stores at 1731435286302 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435286303 (+1 ms)Cleaning up temporary data from old regions at 1731435286318 (+15 ms)Running coprocessor post-open hooks at 1731435286333 (+15 ms)Region opened successfully at 1731435286333 2024-11-12T18:14:46,333 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1731435284929.394b0963ad0828345e14c3be23cb7790., pid=36, masterSystemTime=1731435286059 2024-11-12T18:14:46,333 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1731435284929.6a6d3052a2b28af2dab58e9247bb22b7. 2024-11-12T18:14:46,334 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,jjj,1731435284929.6a6d3052a2b28af2dab58e9247bb22b7. 2024-11-12T18:14:46,334 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,www,1731435284929.ec76766d593d312811f043efb129a5a3. 2024-11-12T18:14:46,334 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=6a6d3052a2b28af2dab58e9247bb22b7, regionState=OPEN, openSeqNum=2, regionLocation=9911683f163c,34847,1731435282512 2024-11-12T18:14:46,334 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => ec76766d593d312811f043efb129a5a3, NAME => 'test,www,1731435284929.ec76766d593d312811f043efb129a5a3.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-12T18:14:46,334 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ec76766d593d312811f043efb129a5a3 2024-11-12T18:14:46,335 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,www,1731435284929.ec76766d593d312811f043efb129a5a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:46,335 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for ec76766d593d312811f043efb129a5a3 2024-11-12T18:14:46,335 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for ec76766d593d312811f043efb129a5a3 2024-11-12T18:14:46,336 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1731435284929.9fbe24cda440bbf727cd581ab33ad7a9., pid=43, masterSystemTime=1731435286106 2024-11-12T18:14:46,338 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36779 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=9911683f163c,34847,1731435282512, table=test, region=6a6d3052a2b28af2dab58e9247bb22b7. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-12T18:14:46,338 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6a6d3052a2b28af2dab58e9247bb22b7, server=9911683f163c,34847,1731435282512 because future has completed 2024-11-12T18:14:46,338 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1731435284929.394b0963ad0828345e14c3be23cb7790. 2024-11-12T18:14:46,338 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,lll,1731435284929.394b0963ad0828345e14c3be23cb7790. 2024-11-12T18:14:46,340 INFO [StoreOpener-ec76766d593d312811f043efb129a5a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ec76766d593d312811f043efb129a5a3 2024-11-12T18:14:46,343 INFO [StoreOpener-ec76766d593d312811f043efb129a5a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec76766d593d312811f043efb129a5a3 columnFamilyName family 2024-11-12T18:14:46,343 DEBUG [StoreOpener-ec76766d593d312811f043efb129a5a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:46,345 INFO [StoreOpener-ec76766d593d312811f043efb129a5a3-1 {}] regionserver.HStore(327): Store=ec76766d593d312811f043efb129a5a3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:46,345 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for ec76766d593d312811f043efb129a5a3 2024-11-12T18:14:46,346 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=14 2024-11-12T18:14:46,346 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 6a6d3052a2b28af2dab58e9247bb22b7, server=9911683f163c,34847,1731435282512 in 302 msec 2024-11-12T18:14:46,346 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/ec76766d593d312811f043efb129a5a3 2024-11-12T18:14:46,347 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/ec76766d593d312811f043efb129a5a3 2024-11-12T18:14:46,348 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for ec76766d593d312811f043efb129a5a3 2024-11-12T18:14:46,348 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for ec76766d593d312811f043efb129a5a3 2024-11-12T18:14:46,348 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1731435284929.9fbe24cda440bbf727cd581ab33ad7a9. 2024-11-12T18:14:46,348 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,ppp,1731435284929.9fbe24cda440bbf727cd581ab33ad7a9. 2024-11-12T18:14:46,348 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,kkk,1731435284929.d59e820eb0eef2601f5990d32a8adbc9. 2024-11-12T18:14:46,349 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => d59e820eb0eef2601f5990d32a8adbc9, NAME => 'test,kkk,1731435284929.d59e820eb0eef2601f5990d32a8adbc9.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-12T18:14:46,349 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d59e820eb0eef2601f5990d32a8adbc9 2024-11-12T18:14:46,349 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36779 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=9911683f163c,45243,1731435282573, table=test, region=0178ff26f4e2d0ab40d50fe6fcd026be. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-12T18:14:46,349 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,kkk,1731435284929.d59e820eb0eef2601f5990d32a8adbc9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:46,350 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for d59e820eb0eef2601f5990d32a8adbc9 2024-11-12T18:14:46,350 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for d59e820eb0eef2601f5990d32a8adbc9 2024-11-12T18:14:46,350 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6a6d3052a2b28af2dab58e9247bb22b7, ASSIGN in 612 msec 2024-11-12T18:14:46,351 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for ec76766d593d312811f043efb129a5a3 2024-11-12T18:14:46,351 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=9fbe24cda440bbf727cd581ab33ad7a9, regionState=OPEN, openSeqNum=2, regionLocation=9911683f163c,40543,1731435282322 2024-11-12T18:14:46,352 INFO [StoreOpener-d59e820eb0eef2601f5990d32a8adbc9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d59e820eb0eef2601f5990d32a8adbc9 2024-11-12T18:14:46,355 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9fbe24cda440bbf727cd581ab33ad7a9, server=9911683f163c,40543,1731435282322 because future has completed 2024-11-12T18:14:46,356 INFO [StoreOpener-d59e820eb0eef2601f5990d32a8adbc9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d59e820eb0eef2601f5990d32a8adbc9 columnFamilyName family 2024-11-12T18:14:46,356 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=394b0963ad0828345e14c3be23cb7790, regionState=OPEN, openSeqNum=2, regionLocation=9911683f163c,45243,1731435282573 2024-11-12T18:14:46,356 DEBUG [StoreOpener-d59e820eb0eef2601f5990d32a8adbc9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:46,357 INFO [StoreOpener-d59e820eb0eef2601f5990d32a8adbc9-1 {}] regionserver.HStore(327): Store=d59e820eb0eef2601f5990d32a8adbc9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:46,359 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for d59e820eb0eef2601f5990d32a8adbc9 2024-11-12T18:14:46,359 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 394b0963ad0828345e14c3be23cb7790, server=9911683f163c,45243,1731435282573 because future has completed 2024-11-12T18:14:46,360 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/d59e820eb0eef2601f5990d32a8adbc9 2024-11-12T18:14:46,361 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/d59e820eb0eef2601f5990d32a8adbc9 2024-11-12T18:14:46,362 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/ec76766d593d312811f043efb129a5a3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T18:14:46,362 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for d59e820eb0eef2601f5990d32a8adbc9 2024-11-12T18:14:46,362 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for d59e820eb0eef2601f5990d32a8adbc9 2024-11-12T18:14:46,363 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened ec76766d593d312811f043efb129a5a3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68127032, jitterRate=0.01517188549041748}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T18:14:46,363 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ec76766d593d312811f043efb129a5a3 2024-11-12T18:14:46,363 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for ec76766d593d312811f043efb129a5a3: Running coprocessor pre-open hook at 1731435286335Writing region info on filesystem at 1731435286335Initializing all the Stores at 1731435286339 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435286339Cleaning up temporary data from old regions at 1731435286348 (+9 ms)Running coprocessor post-open hooks at 1731435286363 (+15 ms)Region opened successfully at 1731435286363 2024-11-12T18:14:46,365 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1731435284929.ec76766d593d312811f043efb129a5a3., pid=47, masterSystemTime=1731435286147 2024-11-12T18:14:46,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=20 2024-11-12T18:14:46,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 9fbe24cda440bbf727cd581ab33ad7a9, server=9911683f163c,40543,1731435282322 in 388 msec 2024-11-12T18:14:46,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN in 632 msec 2024-11-12T18:14:46,376 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1731435284929.ec76766d593d312811f043efb129a5a3. 2024-11-12T18:14:46,376 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,www,1731435284929.ec76766d593d312811f043efb129a5a3. 2024-11-12T18:14:46,376 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,ttt,1731435284929.4e38ac98b32f350d84df37b6915c56a3. 2024-11-12T18:14:46,376 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => 4e38ac98b32f350d84df37b6915c56a3, NAME => 'test,ttt,1731435284929.4e38ac98b32f350d84df37b6915c56a3.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-12T18:14:46,377 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4e38ac98b32f350d84df37b6915c56a3 2024-11-12T18:14:46,378 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,ttt,1731435284929.4e38ac98b32f350d84df37b6915c56a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:46,378 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for 4e38ac98b32f350d84df37b6915c56a3 2024-11-12T18:14:46,378 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for 4e38ac98b32f350d84df37b6915c56a3 2024-11-12T18:14:46,382 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=16 2024-11-12T18:14:46,382 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 394b0963ad0828345e14c3be23cb7790, server=9911683f163c,45243,1731435282573 in 438 msec 2024-11-12T18:14:46,386 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=394b0963ad0828345e14c3be23cb7790, ASSIGN in 648 msec 2024-11-12T18:14:46,387 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for d59e820eb0eef2601f5990d32a8adbc9 2024-11-12T18:14:46,388 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=ec76766d593d312811f043efb129a5a3, regionState=OPEN, openSeqNum=2, regionLocation=9911683f163c,34847,1731435282512 2024-11-12T18:14:46,391 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure ec76766d593d312811f043efb129a5a3, server=9911683f163c,34847,1731435282512 because future has completed 2024-11-12T18:14:46,393 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a7b0558136890d45b56827e5af943cfa, ASSIGN in 656 msec 2024-11-12T18:14:46,397 INFO [StoreOpener-4e38ac98b32f350d84df37b6915c56a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4e38ac98b32f350d84df37b6915c56a3 2024-11-12T18:14:46,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=27 2024-11-12T18:14:46,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure ec76766d593d312811f043efb129a5a3, server=9911683f163c,34847,1731435282512 in 401 msec 2024-11-12T18:14:46,399 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ec76766d593d312811f043efb129a5a3, ASSIGN in 663 msec 2024-11-12T18:14:46,400 INFO [StoreOpener-4e38ac98b32f350d84df37b6915c56a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e38ac98b32f350d84df37b6915c56a3 columnFamilyName family 2024-11-12T18:14:46,400 DEBUG [StoreOpener-4e38ac98b32f350d84df37b6915c56a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:46,401 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=4b703e982cd6bbf72ec2f771b120c649, regionState=OPEN, openSeqNum=2, regionLocation=9911683f163c,40543,1731435282322 2024-11-12T18:14:46,402 INFO [StoreOpener-4e38ac98b32f350d84df37b6915c56a3-1 {}] regionserver.HStore(327): Store=4e38ac98b32f350d84df37b6915c56a3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:46,403 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for 4e38ac98b32f350d84df37b6915c56a3 2024-11-12T18:14:46,404 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/4e38ac98b32f350d84df37b6915c56a3 2024-11-12T18:14:46,404 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,40543,1731435282322 because future has completed 2024-11-12T18:14:46,405 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/4e38ac98b32f350d84df37b6915c56a3 2024-11-12T18:14:46,405 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for 4e38ac98b32f350d84df37b6915c56a3 2024-11-12T18:14:46,405 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for 4e38ac98b32f350d84df37b6915c56a3 2024-11-12T18:14:46,407 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=dff0a9c660bd5084b34753ea40ea3278, regionState=OPEN, openSeqNum=2, regionLocation=9911683f163c,45243,1731435282573 2024-11-12T18:14:46,408 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for 4e38ac98b32f350d84df37b6915c56a3 2024-11-12T18:14:46,419 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/d59e820eb0eef2601f5990d32a8adbc9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T18:14:46,422 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened d59e820eb0eef2601f5990d32a8adbc9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71249715, jitterRate=0.06170348823070526}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T18:14:46,422 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d59e820eb0eef2601f5990d32a8adbc9 2024-11-12T18:14:46,422 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for d59e820eb0eef2601f5990d32a8adbc9: Running coprocessor pre-open hook at 1731435286350Writing region info on filesystem at 1731435286350Initializing all the Stores at 1731435286352 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435286352Cleaning up temporary data from old regions at 1731435286362 (+10 ms)Running coprocessor post-open hooks at 1731435286422 (+60 ms)Region opened successfully at 1731435286422 2024-11-12T18:14:46,425 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure dff0a9c660bd5084b34753ea40ea3278, server=9911683f163c,45243,1731435282573 because future has completed 2024-11-12T18:14:46,429 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1731435284929.d59e820eb0eef2601f5990d32a8adbc9., pid=42, masterSystemTime=1731435286106 2024-11-12T18:14:46,440 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/4e38ac98b32f350d84df37b6915c56a3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T18:14:46,443 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1731435284929.d59e820eb0eef2601f5990d32a8adbc9. 2024-11-12T18:14:46,444 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,kkk,1731435284929.d59e820eb0eef2601f5990d32a8adbc9. 2024-11-12T18:14:46,444 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,ddd,1731435284929.6e290ad15dec09a0cb11782988af7c4c. 2024-11-12T18:14:46,444 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 6e290ad15dec09a0cb11782988af7c4c, NAME => 'test,ddd,1731435284929.6e290ad15dec09a0cb11782988af7c4c.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-12T18:14:46,445 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6e290ad15dec09a0cb11782988af7c4c 2024-11-12T18:14:46,445 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,ddd,1731435284929.6e290ad15dec09a0cb11782988af7c4c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:46,445 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 6e290ad15dec09a0cb11782988af7c4c 2024-11-12T18:14:46,445 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 6e290ad15dec09a0cb11782988af7c4c 2024-11-12T18:14:46,447 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened 4e38ac98b32f350d84df37b6915c56a3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65019784, jitterRate=-0.03112971782684326}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T18:14:46,447 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4e38ac98b32f350d84df37b6915c56a3 2024-11-12T18:14:46,447 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for 4e38ac98b32f350d84df37b6915c56a3: Running coprocessor pre-open hook at 1731435286378Writing region info on filesystem at 1731435286378Initializing all the Stores at 1731435286386 (+8 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435286386Cleaning up temporary data from old regions at 1731435286405 (+19 ms)Running coprocessor post-open hooks at 1731435286447 (+42 ms)Region opened successfully at 1731435286447 2024-11-12T18:14:46,449 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1731435284929.4e38ac98b32f350d84df37b6915c56a3., pid=49, masterSystemTime=1731435286147 2024-11-12T18:14:46,450 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=18 2024-11-12T18:14:46,450 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,40543,1731435282322 in 474 msec 2024-11-12T18:14:46,453 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1731435284929.4e38ac98b32f350d84df37b6915c56a3. 2024-11-12T18:14:46,453 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,ttt,1731435284929.4e38ac98b32f350d84df37b6915c56a3. 2024-11-12T18:14:46,454 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,zzz,1731435284929.e4348dabb8addd0673ec5266827d8b4d. 2024-11-12T18:14:46,454 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => e4348dabb8addd0673ec5266827d8b4d, NAME => 'test,zzz,1731435284929.e4348dabb8addd0673ec5266827d8b4d.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-12T18:14:46,454 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e4348dabb8addd0673ec5266827d8b4d 2024-11-12T18:14:46,454 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,zzz,1731435284929.e4348dabb8addd0673ec5266827d8b4d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:46,455 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for e4348dabb8addd0673ec5266827d8b4d 2024-11-12T18:14:46,455 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for e4348dabb8addd0673ec5266827d8b4d 2024-11-12T18:14:46,456 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36779 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=9911683f163c,45243,1731435282573, table=test, region=dff0a9c660bd5084b34753ea40ea3278. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-12T18:14:46,456 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36779 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=9911683f163c,45243,1731435282573, table=test, region=0178ff26f4e2d0ab40d50fe6fcd026be. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-12T18:14:46,456 INFO [StoreOpener-6e290ad15dec09a0cb11782988af7c4c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6e290ad15dec09a0cb11782988af7c4c 2024-11-12T18:14:46,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN in 717 msec 2024-11-12T18:14:46,461 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=4e38ac98b32f350d84df37b6915c56a3, regionState=OPEN, openSeqNum=2, regionLocation=9911683f163c,34847,1731435282512 2024-11-12T18:14:46,461 INFO [StoreOpener-6e290ad15dec09a0cb11782988af7c4c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6e290ad15dec09a0cb11782988af7c4c columnFamilyName family 2024-11-12T18:14:46,461 DEBUG [StoreOpener-6e290ad15dec09a0cb11782988af7c4c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:46,463 INFO [StoreOpener-6e290ad15dec09a0cb11782988af7c4c-1 {}] regionserver.HStore(327): Store=6e290ad15dec09a0cb11782988af7c4c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:46,463 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 6e290ad15dec09a0cb11782988af7c4c 2024-11-12T18:14:46,465 INFO [StoreOpener-e4348dabb8addd0673ec5266827d8b4d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e4348dabb8addd0673ec5266827d8b4d 2024-11-12T18:14:46,469 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/6e290ad15dec09a0cb11782988af7c4c 2024-11-12T18:14:46,470 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e38ac98b32f350d84df37b6915c56a3, server=9911683f163c,34847,1731435282512 because future has completed 2024-11-12T18:14:46,470 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/6e290ad15dec09a0cb11782988af7c4c 2024-11-12T18:14:46,473 INFO [StoreOpener-e4348dabb8addd0673ec5266827d8b4d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e4348dabb8addd0673ec5266827d8b4d columnFamilyName family 2024-11-12T18:14:46,474 DEBUG [StoreOpener-e4348dabb8addd0673ec5266827d8b4d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:46,475 INFO [StoreOpener-e4348dabb8addd0673ec5266827d8b4d-1 {}] regionserver.HStore(327): Store=e4348dabb8addd0673ec5266827d8b4d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:46,475 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for e4348dabb8addd0673ec5266827d8b4d 2024-11-12T18:14:46,476 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/e4348dabb8addd0673ec5266827d8b4d 2024-11-12T18:14:46,476 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 6e290ad15dec09a0cb11782988af7c4c 2024-11-12T18:14:46,477 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 6e290ad15dec09a0cb11782988af7c4c 2024-11-12T18:14:46,478 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/e4348dabb8addd0673ec5266827d8b4d 2024-11-12T18:14:46,478 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=24 2024-11-12T18:14:46,478 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 4e38ac98b32f350d84df37b6915c56a3, server=9911683f163c,34847,1731435282512 in 471 msec 2024-11-12T18:14:46,479 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for e4348dabb8addd0673ec5266827d8b4d 2024-11-12T18:14:46,479 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for e4348dabb8addd0673ec5266827d8b4d 2024-11-12T18:14:46,481 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4e38ac98b32f350d84df37b6915c56a3, ASSIGN in 744 msec 2024-11-12T18:14:46,483 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=d59e820eb0eef2601f5990d32a8adbc9, regionState=OPEN, openSeqNum=2, regionLocation=9911683f163c,40543,1731435282322 2024-11-12T18:14:46,484 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for e4348dabb8addd0673ec5266827d8b4d 2024-11-12T18:14:46,486 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 6e290ad15dec09a0cb11782988af7c4c 2024-11-12T18:14:46,488 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure d59e820eb0eef2601f5990d32a8adbc9, server=9911683f163c,40543,1731435282322 because future has completed 2024-11-12T18:14:46,490 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/e4348dabb8addd0673ec5266827d8b4d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T18:14:46,491 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened e4348dabb8addd0673ec5266827d8b4d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59227163, jitterRate=-0.11744649708271027}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T18:14:46,491 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e4348dabb8addd0673ec5266827d8b4d 2024-11-12T18:14:46,491 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for e4348dabb8addd0673ec5266827d8b4d: Running coprocessor pre-open hook at 1731435286455Writing region info on filesystem at 1731435286455Initializing all the Stores at 1731435286460 (+5 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435286460Cleaning up temporary data from old regions at 1731435286479 (+19 ms)Running coprocessor post-open hooks at 1731435286491 (+12 ms)Region opened successfully at 1731435286491 2024-11-12T18:14:46,497 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1731435284929.e4348dabb8addd0673ec5266827d8b4d., pid=50, masterSystemTime=1731435286147 2024-11-12T18:14:46,497 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=12 2024-11-12T18:14:46,497 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure dff0a9c660bd5084b34753ea40ea3278, server=9911683f163c,45243,1731435282573 in 571 msec 2024-11-12T18:14:46,500 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1731435284929.e4348dabb8addd0673ec5266827d8b4d. 2024-11-12T18:14:46,500 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,zzz,1731435284929.e4348dabb8addd0673ec5266827d8b4d. 2024-11-12T18:14:46,500 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,ccc,1731435284929.3a62a194de5221e9f84b2cfd2e83a46a. 2024-11-12T18:14:46,500 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => 3a62a194de5221e9f84b2cfd2e83a46a, NAME => 'test,ccc,1731435284929.3a62a194de5221e9f84b2cfd2e83a46a.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-12T18:14:46,500 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=dff0a9c660bd5084b34753ea40ea3278, ASSIGN in 764 msec 2024-11-12T18:14:46,501 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 3a62a194de5221e9f84b2cfd2e83a46a 2024-11-12T18:14:46,501 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,ccc,1731435284929.3a62a194de5221e9f84b2cfd2e83a46a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:46,501 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for 3a62a194de5221e9f84b2cfd2e83a46a 2024-11-12T18:14:46,501 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for 3a62a194de5221e9f84b2cfd2e83a46a 2024-11-12T18:14:46,502 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=e4348dabb8addd0673ec5266827d8b4d, regionState=OPEN, openSeqNum=2, regionLocation=9911683f163c,34847,1731435282512 2024-11-12T18:14:46,505 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure e4348dabb8addd0673ec5266827d8b4d, server=9911683f163c,34847,1731435282512 because future has completed 2024-11-12T18:14:46,506 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/6e290ad15dec09a0cb11782988af7c4c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T18:14:46,506 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 6e290ad15dec09a0cb11782988af7c4c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64958540, jitterRate=-0.03204232454299927}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T18:14:46,506 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6e290ad15dec09a0cb11782988af7c4c 2024-11-12T18:14:46,506 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 6e290ad15dec09a0cb11782988af7c4c: Running coprocessor pre-open hook at 1731435286445Writing region info on filesystem at 1731435286446 (+1 ms)Initializing all the Stores at 1731435286448 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435286448Cleaning up temporary data from old regions at 1731435286477 (+29 ms)Running coprocessor post-open hooks at 1731435286506 (+29 ms)Region opened successfully at 1731435286506 2024-11-12T18:14:46,508 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1731435284929.6e290ad15dec09a0cb11782988af7c4c., pid=46, masterSystemTime=1731435286106 2024-11-12T18:14:46,508 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=15 2024-11-12T18:14:46,508 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure d59e820eb0eef2601f5990d32a8adbc9, server=9911683f163c,40543,1731435282322 in 538 msec 2024-11-12T18:14:46,509 INFO [StoreOpener-3a62a194de5221e9f84b2cfd2e83a46a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 3a62a194de5221e9f84b2cfd2e83a46a 2024-11-12T18:14:46,510 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1731435284929.6e290ad15dec09a0cb11782988af7c4c. 2024-11-12T18:14:46,510 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,ddd,1731435284929.6e290ad15dec09a0cb11782988af7c4c. 2024-11-12T18:14:46,511 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,ggg,1731435284929.26fc65f8daf71d5831131b1bc6746a63. 2024-11-12T18:14:46,511 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 26fc65f8daf71d5831131b1bc6746a63, NAME => 'test,ggg,1731435284929.26fc65f8daf71d5831131b1bc6746a63.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-12T18:14:46,511 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 26fc65f8daf71d5831131b1bc6746a63 2024-11-12T18:14:46,511 INFO [StoreOpener-3a62a194de5221e9f84b2cfd2e83a46a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3a62a194de5221e9f84b2cfd2e83a46a columnFamilyName family 2024-11-12T18:14:46,511 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,ggg,1731435284929.26fc65f8daf71d5831131b1bc6746a63.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:46,511 DEBUG [StoreOpener-3a62a194de5221e9f84b2cfd2e83a46a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:46,512 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 26fc65f8daf71d5831131b1bc6746a63 2024-11-12T18:14:46,512 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 26fc65f8daf71d5831131b1bc6746a63 2024-11-12T18:14:46,512 INFO [StoreOpener-3a62a194de5221e9f84b2cfd2e83a46a-1 {}] regionserver.HStore(327): Store=3a62a194de5221e9f84b2cfd2e83a46a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:46,512 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for 3a62a194de5221e9f84b2cfd2e83a46a 2024-11-12T18:14:46,513 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN in 774 msec 2024-11-12T18:14:46,514 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/3a62a194de5221e9f84b2cfd2e83a46a 2024-11-12T18:14:46,514 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=6e290ad15dec09a0cb11782988af7c4c, regionState=OPEN, openSeqNum=2, regionLocation=9911683f163c,40543,1731435282322 2024-11-12T18:14:46,514 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/3a62a194de5221e9f84b2cfd2e83a46a 2024-11-12T18:14:46,515 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for 3a62a194de5221e9f84b2cfd2e83a46a 2024-11-12T18:14:46,515 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for 3a62a194de5221e9f84b2cfd2e83a46a 2024-11-12T18:14:46,518 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for 3a62a194de5221e9f84b2cfd2e83a46a 2024-11-12T18:14:46,521 INFO [StoreOpener-26fc65f8daf71d5831131b1bc6746a63-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 26fc65f8daf71d5831131b1bc6746a63 2024-11-12T18:14:46,522 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6e290ad15dec09a0cb11782988af7c4c, server=9911683f163c,40543,1731435282322 because future has completed 2024-11-12T18:14:46,524 INFO [StoreOpener-26fc65f8daf71d5831131b1bc6746a63-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 26fc65f8daf71d5831131b1bc6746a63 columnFamilyName family 2024-11-12T18:14:46,524 DEBUG [StoreOpener-26fc65f8daf71d5831131b1bc6746a63-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:46,525 INFO [StoreOpener-26fc65f8daf71d5831131b1bc6746a63-1 {}] regionserver.HStore(327): Store=26fc65f8daf71d5831131b1bc6746a63/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:46,525 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=30 2024-11-12T18:14:46,525 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/3a62a194de5221e9f84b2cfd2e83a46a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T18:14:46,525 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure e4348dabb8addd0673ec5266827d8b4d, server=9911683f163c,34847,1731435282512 in 514 msec 2024-11-12T18:14:46,525 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 26fc65f8daf71d5831131b1bc6746a63 2024-11-12T18:14:46,527 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened 3a62a194de5221e9f84b2cfd2e83a46a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69201908, jitterRate=0.031188786029815674}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T18:14:46,527 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3a62a194de5221e9f84b2cfd2e83a46a 2024-11-12T18:14:46,527 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for 3a62a194de5221e9f84b2cfd2e83a46a: Running coprocessor pre-open hook at 1731435286501Writing region info on filesystem at 1731435286501Initializing all the Stores at 1731435286503 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435286503Cleaning up temporary data from old regions at 1731435286515 (+12 ms)Running coprocessor post-open hooks at 1731435286527 (+12 ms)Region opened successfully at 1731435286527 2024-11-12T18:14:46,528 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1731435284929.3a62a194de5221e9f84b2cfd2e83a46a., pid=53, masterSystemTime=1731435286147 2024-11-12T18:14:46,528 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/26fc65f8daf71d5831131b1bc6746a63 2024-11-12T18:14:46,529 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/26fc65f8daf71d5831131b1bc6746a63 2024-11-12T18:14:46,530 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 26fc65f8daf71d5831131b1bc6746a63 2024-11-12T18:14:46,530 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 26fc65f8daf71d5831131b1bc6746a63 2024-11-12T18:14:46,530 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e4348dabb8addd0673ec5266827d8b4d, ASSIGN in 791 msec 2024-11-12T18:14:46,531 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1731435284929.3a62a194de5221e9f84b2cfd2e83a46a. 2024-11-12T18:14:46,531 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,ccc,1731435284929.3a62a194de5221e9f84b2cfd2e83a46a. 2024-11-12T18:14:46,532 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,fff,1731435284929.4aff96e33c9923b5d2448f72171fd310. 2024-11-12T18:14:46,532 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => 4aff96e33c9923b5d2448f72171fd310, NAME => 'test,fff,1731435284929.4aff96e33c9923b5d2448f72171fd310.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-12T18:14:46,533 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 26fc65f8daf71d5831131b1bc6746a63 2024-11-12T18:14:46,533 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4aff96e33c9923b5d2448f72171fd310 2024-11-12T18:14:46,533 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,fff,1731435284929.4aff96e33c9923b5d2448f72171fd310.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:46,533 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for 4aff96e33c9923b5d2448f72171fd310 2024-11-12T18:14:46,533 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for 4aff96e33c9923b5d2448f72171fd310 2024-11-12T18:14:46,536 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/26fc65f8daf71d5831131b1bc6746a63/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T18:14:46,537 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 26fc65f8daf71d5831131b1bc6746a63; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65678801, jitterRate=-0.02130959928035736}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T18:14:46,537 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 26fc65f8daf71d5831131b1bc6746a63 2024-11-12T18:14:46,537 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 26fc65f8daf71d5831131b1bc6746a63: Running coprocessor pre-open hook at 1731435286512Writing region info on filesystem at 1731435286512Initializing all the Stores at 1731435286513 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435286513Cleaning up temporary data from old regions at 1731435286530 (+17 ms)Running coprocessor post-open hooks at 1731435286537 (+7 ms)Region opened successfully at 1731435286537 2024-11-12T18:14:46,539 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1731435284929.26fc65f8daf71d5831131b1bc6746a63., pid=41, masterSystemTime=1731435286106 2024-11-12T18:14:46,539 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=8 2024-11-12T18:14:46,539 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 6e290ad15dec09a0cb11782988af7c4c, server=9911683f163c,40543,1731435282322 in 545 msec 2024-11-12T18:14:46,541 INFO [StoreOpener-4aff96e33c9923b5d2448f72171fd310-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4aff96e33c9923b5d2448f72171fd310 2024-11-12T18:14:46,542 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1731435284929.26fc65f8daf71d5831131b1bc6746a63. 2024-11-12T18:14:46,542 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,ggg,1731435284929.26fc65f8daf71d5831131b1bc6746a63. 2024-11-12T18:14:46,542 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6e290ad15dec09a0cb11782988af7c4c, ASSIGN in 805 msec 2024-11-12T18:14:46,544 INFO [StoreOpener-4aff96e33c9923b5d2448f72171fd310-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4aff96e33c9923b5d2448f72171fd310 columnFamilyName family 2024-11-12T18:14:46,544 DEBUG [StoreOpener-4aff96e33c9923b5d2448f72171fd310-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:46,544 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=26fc65f8daf71d5831131b1bc6746a63, regionState=OPEN, openSeqNum=2, regionLocation=9911683f163c,40543,1731435282322 2024-11-12T18:14:46,546 INFO [StoreOpener-4aff96e33c9923b5d2448f72171fd310-1 {}] regionserver.HStore(327): Store=4aff96e33c9923b5d2448f72171fd310/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:46,547 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for 4aff96e33c9923b5d2448f72171fd310 2024-11-12T18:14:46,548 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26fc65f8daf71d5831131b1bc6746a63, server=9911683f163c,40543,1731435282322 because future has completed 2024-11-12T18:14:46,548 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/4aff96e33c9923b5d2448f72171fd310 2024-11-12T18:14:46,549 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/4aff96e33c9923b5d2448f72171fd310 2024-11-12T18:14:46,549 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=3a62a194de5221e9f84b2cfd2e83a46a, regionState=OPEN, openSeqNum=2, regionLocation=9911683f163c,34847,1731435282512 2024-11-12T18:14:46,550 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for 4aff96e33c9923b5d2448f72171fd310 2024-11-12T18:14:46,550 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for 4aff96e33c9923b5d2448f72171fd310 2024-11-12T18:14:46,552 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3a62a194de5221e9f84b2cfd2e83a46a, server=9911683f163c,34847,1731435282512 because future has completed 2024-11-12T18:14:46,552 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36779 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=9911683f163c,40543,1731435282322, table=test, region=26fc65f8daf71d5831131b1bc6746a63. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-12T18:14:46,554 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for 4aff96e33c9923b5d2448f72171fd310 2024-11-12T18:14:46,556 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=11 2024-11-12T18:14:46,556 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 26fc65f8daf71d5831131b1bc6746a63, server=9911683f163c,40543,1731435282322 in 590 msec 2024-11-12T18:14:46,557 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/4aff96e33c9923b5d2448f72171fd310/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T18:14:46,558 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened 4aff96e33c9923b5d2448f72171fd310; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71202705, jitterRate=0.06100298464298248}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T18:14:46,558 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4aff96e33c9923b5d2448f72171fd310 2024-11-12T18:14:46,558 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN in 822 msec 2024-11-12T18:14:46,558 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for 4aff96e33c9923b5d2448f72171fd310: Running coprocessor pre-open hook at 1731435286534Writing region info on filesystem at 1731435286534Initializing all the Stores at 1731435286535 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435286536 (+1 ms)Cleaning up temporary data from old regions at 1731435286550 (+14 ms)Running coprocessor post-open hooks at 1731435286558 (+8 ms)Region opened successfully at 1731435286558 2024-11-12T18:14:46,559 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1731435284929.4aff96e33c9923b5d2448f72171fd310., pid=54, masterSystemTime=1731435286147 2024-11-12T18:14:46,563 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1731435284929.4aff96e33c9923b5d2448f72171fd310. 2024-11-12T18:14:46,563 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,fff,1731435284929.4aff96e33c9923b5d2448f72171fd310. 2024-11-12T18:14:46,563 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36779 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=9911683f163c,45243,1731435282573, table=test, region=0178ff26f4e2d0ab40d50fe6fcd026be. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-12T18:14:46,564 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=7 2024-11-12T18:14:46,564 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 3a62a194de5221e9f84b2cfd2e83a46a, server=9911683f163c,34847,1731435282512 in 534 msec 2024-11-12T18:14:46,566 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=3a62a194de5221e9f84b2cfd2e83a46a, ASSIGN in 830 msec 2024-11-12T18:14:46,567 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=4aff96e33c9923b5d2448f72171fd310, regionState=OPEN, openSeqNum=2, regionLocation=9911683f163c,34847,1731435282512 2024-11-12T18:14:46,570 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4aff96e33c9923b5d2448f72171fd310, server=9911683f163c,34847,1731435282512 because future has completed 2024-11-12T18:14:46,571 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=2248a0d1296d1dbe4f8c9805798db03c, regionState=OPEN, openSeqNum=2, regionLocation=9911683f163c,40543,1731435282322 2024-11-12T18:14:46,573 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2248a0d1296d1dbe4f8c9805798db03c, server=9911683f163c,40543,1731435282322 because future has completed 2024-11-12T18:14:46,577 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=10 2024-11-12T18:14:46,577 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 4aff96e33c9923b5d2448f72171fd310, server=9911683f163c,34847,1731435282512 in 545 msec 2024-11-12T18:14:46,579 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4aff96e33c9923b5d2448f72171fd310, ASSIGN in 843 msec 2024-11-12T18:14:46,583 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=6 2024-11-12T18:14:46,583 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 2248a0d1296d1dbe4f8c9805798db03c, server=9911683f163c,40543,1731435282322 in 569 msec 2024-11-12T18:14:46,585 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN in 849 msec 2024-11-12T18:14:46,588 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=28 2024-11-12T18:14:46,588 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure 0178ff26f4e2d0ab40d50fe6fcd026be, server=9911683f163c,45243,1731435282573 in 680 msec 2024-11-12T18:14:46,591 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=4 2024-11-12T18:14:46,591 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=0178ff26f4e2d0ab40d50fe6fcd026be, ASSIGN in 854 msec 2024-11-12T18:14:46,593 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T18:14:46,593 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731435286593"}]},"ts":"1731435286593"} 2024-11-12T18:14:46,596 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-11-12T18:14:46,598 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T18:14:46,605 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 1.6360 sec 2024-11-12T18:14:47,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T18:14:47,148 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-11-12T18:14:47,148 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-11-12T18:14:47,150 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T18:14:47,170 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-11-12T18:14:47,170 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T18:14:47,171 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-11-12T18:14:47,172 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T18:14:47,177 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-12T18:14:47,197 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-12T18:14:47,211 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:14:47,213 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:14:47,217 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:14:47,219 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45438, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:14:47,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 3a62a194de5221e9f84b2cfd2e83a46a changed from -1.0 to 0.0, refreshing cache 2024-11-12T18:14:47,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4aff96e33c9923b5d2448f72171fd310 changed from -1.0 to 0.0, refreshing cache 2024-11-12T18:14:47,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 96e9ca1050c6b52ea6556fdb4b5d5f39 changed from -1.0 to 0.0, refreshing cache 2024-11-12T18:14:47,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6a6d3052a2b28af2dab58e9247bb22b7 changed from -1.0 to 0.0, refreshing cache 2024-11-12T18:14:47,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c739eeba765df343af4ef63444655c9f changed from -1.0 to 0.0, refreshing cache 2024-11-12T18:14:47,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6d7c21378c2e7788ac010db26909da83 changed from -1.0 to 0.0, refreshing cache 2024-11-12T18:14:47,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4e38ac98b32f350d84df37b6915c56a3 changed from -1.0 to 0.0, refreshing cache 2024-11-12T18:14:47,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ec76766d593d312811f043efb129a5a3 changed from -1.0 to 0.0, refreshing cache 2024-11-12T18:14:47,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e4348dabb8addd0673ec5266827d8b4d changed from -1.0 to 0.0, refreshing cache 2024-11-12T18:14:47,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2248a0d1296d1dbe4f8c9805798db03c changed from -1.0 to 0.0, refreshing cache 2024-11-12T18:14:47,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6e290ad15dec09a0cb11782988af7c4c changed from -1.0 to 0.0, refreshing cache 2024-11-12T18:14:47,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 26fc65f8daf71d5831131b1bc6746a63 changed from -1.0 to 0.0, refreshing cache 2024-11-12T18:14:47,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d59e820eb0eef2601f5990d32a8adbc9 changed from -1.0 to 0.0, refreshing cache 2024-11-12T18:14:47,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4b703e982cd6bbf72ec2f771b120c649 changed from -1.0 to 0.0, refreshing cache 2024-11-12T18:14:47,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 9fbe24cda440bbf727cd581ab33ad7a9 changed from -1.0 to 0.0, refreshing cache 2024-11-12T18:14:47,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1b81a9be48fbe50dde523592a87a4611 changed from -1.0 to 0.0, refreshing cache 2024-11-12T18:14:47,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 56e168ba1a3103978bee5b6c43feba36 changed from -1.0 to 0.0, refreshing cache 2024-11-12T18:14:47,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region fde2b7fbe9fa7c8a87ab544f5542d82a changed from -1.0 to 0.0, refreshing cache 2024-11-12T18:14:47,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5dcfe82d27e00f564e4ad71ea2e78f5c changed from -1.0 to 0.0, refreshing cache 2024-11-12T18:14:47,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c0b5ce997df05cef2b0f3fd25e49fafc changed from -1.0 to 0.0, refreshing cache 2024-11-12T18:14:47,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region dff0a9c660bd5084b34753ea40ea3278 changed from -1.0 to 0.0, refreshing cache 2024-11-12T18:14:47,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 394b0963ad0828345e14c3be23cb7790 changed from -1.0 to 0.0, refreshing cache 2024-11-12T18:14:47,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 40b076d46dbfff2e1f3c89ad5c3f814d changed from -1.0 to 0.0, refreshing cache 2024-11-12T18:14:47,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4795f188f4931ea02df11d7474e4daa4 changed from -1.0 to 0.0, refreshing cache 2024-11-12T18:14:47,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a7b0558136890d45b56827e5af943cfa changed from -1.0 to 0.0, refreshing cache 2024-11-12T18:14:47,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0178ff26f4e2d0ab40d50fe6fcd026be changed from -1.0 to 0.0, refreshing cache 2024-11-12T18:14:47,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-11-12T18:14:47,263 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-11-12T18:14:47,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-11-12T18:14:47,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] balancer.BalancerClusterState(204): Hosts are {9911683f163c=0} racks are {/default-rack=0} 2024-11-12T18:14:47,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] balancer.BalancerClusterState(303): server 0 has 9 regions 2024-11-12T18:14:47,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] balancer.BalancerClusterState(303): server 1 has 10 regions 2024-11-12T18:14:47,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] balancer.BalancerClusterState(303): server 2 has 8 regions 2024-11-12T18:14:47,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-12T18:14:47,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-12T18:14:47,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-12T18:14:47,265 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-12T18:14:47,265 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-12T18:14:47,265 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-12T18:14:47,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-11-12T18:14:47,268 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] balancer.StochasticLoadBalancer(403): Cluster wide - Calculating plan. may take up to 30000ms to complete. 2024-11-12T18:14:47,269 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] balancer.StochasticLoadBalancer(515): Start StochasticLoadBalancer.balancer, initial weighted average imbalance=0.06931633435746164, functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.06415002990995843, need balance); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); computedMaxSteps=64800 2024-11-12T18:14:47,742 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] balancer.StochasticLoadBalancer(562): Finished computing new moving plan. Computation took 476 ms to try 64800 different iterations. Found a solution that moves 1 regions; Going from a computed imbalance of 0.06931633435746164 to a new imbalance of 0.01690158032826896. funtionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.037037037037037035, need balance); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-11-12T18:14:47,746 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] master.HMaster(2167): Balancer plans size is 1, the balance interval is 300000 ms, and the max number regions in transition is 27 2024-11-12T18:14:47,746 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] master.HMaster(2172): balance hri=6e290ad15dec09a0cb11782988af7c4c, source=9911683f163c,40543,1731435282322, destination=9911683f163c,45243,1731435282573 2024-11-12T18:14:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=6e290ad15dec09a0cb11782988af7c4c, REOPEN/MOVE 2024-11-12T18:14:47,748 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=6e290ad15dec09a0cb11782988af7c4c, REOPEN/MOVE 2024-11-12T18:14:47,751 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=6e290ad15dec09a0cb11782988af7c4c, regionState=CLOSING, regionLocation=9911683f163c,40543,1731435282322 2024-11-12T18:14:47,754 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test, region=6e290ad15dec09a0cb11782988af7c4c, REOPEN/MOVE because future has completed 2024-11-12T18:14:47,754 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T18:14:47,755 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6e290ad15dec09a0cb11782988af7c4c, server=9911683f163c,40543,1731435282322}] 2024-11-12T18:14:47,913 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(122): Close 6e290ad15dec09a0cb11782988af7c4c 2024-11-12T18:14:47,913 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T18:14:47,914 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1722): Closing 6e290ad15dec09a0cb11782988af7c4c, disabling compactions & flushes 2024-11-12T18:14:47,914 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1755): Closing region test,ddd,1731435284929.6e290ad15dec09a0cb11782988af7c4c. 2024-11-12T18:14:47,914 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1731435284929.6e290ad15dec09a0cb11782988af7c4c. 2024-11-12T18:14:47,914 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1731435284929.6e290ad15dec09a0cb11782988af7c4c. after waiting 0 ms 2024-11-12T18:14:47,914 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1731435284929.6e290ad15dec09a0cb11782988af7c4c. 2024-11-12T18:14:47,920 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/6e290ad15dec09a0cb11782988af7c4c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T18:14:47,925 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1973): Closed test,ddd,1731435284929.6e290ad15dec09a0cb11782988af7c4c. 2024-11-12T18:14:47,926 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1676): Region close journal for 6e290ad15dec09a0cb11782988af7c4c: Waiting for close lock at 1731435287914Running coprocessor pre-close hooks at 1731435287914Disabling compacts and flushes for region at 1731435287914Disabling writes for close at 1731435287914Writing region close event to WAL at 1731435287914Running coprocessor post-close hooks at 1731435287923 (+9 ms)Closed at 1731435287925 (+2 ms) 2024-11-12T18:14:47,926 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegionServer(3302): Adding 6e290ad15dec09a0cb11782988af7c4c move to 9911683f163c,45243,1731435282573 record at close sequenceid=2 2024-11-12T18:14:47,929 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(157): Closed 6e290ad15dec09a0cb11782988af7c4c 2024-11-12T18:14:47,930 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=6e290ad15dec09a0cb11782988af7c4c, regionState=CLOSED 2024-11-12T18:14:47,933 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6e290ad15dec09a0cb11782988af7c4c, server=9911683f163c,40543,1731435282322 because future has completed 2024-11-12T18:14:47,936 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-12T18:14:47,936 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; CloseRegionProcedure 6e290ad15dec09a0cb11782988af7c4c, server=9911683f163c,40543,1731435282322 in 180 msec 2024-11-12T18:14:47,937 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6e290ad15dec09a0cb11782988af7c4c, REOPEN/MOVE; state=CLOSED, location=9911683f163c,45243,1731435282573; forceNewPlan=false, retain=false 2024-11-12T18:14:48,088 INFO [9911683f163c:36779 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-12T18:14:48,088 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=6e290ad15dec09a0cb11782988af7c4c, regionState=OPENING, regionLocation=9911683f163c,45243,1731435282573 2024-11-12T18:14:48,091 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6e290ad15dec09a0cb11782988af7c4c, REOPEN/MOVE because future has completed 2024-11-12T18:14:48,092 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6e290ad15dec09a0cb11782988af7c4c, server=9911683f163c,45243,1731435282573}] 2024-11-12T18:14:48,250 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open test,ddd,1731435284929.6e290ad15dec09a0cb11782988af7c4c. 2024-11-12T18:14:48,250 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => 6e290ad15dec09a0cb11782988af7c4c, NAME => 'test,ddd,1731435284929.6e290ad15dec09a0cb11782988af7c4c.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-12T18:14:48,250 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6e290ad15dec09a0cb11782988af7c4c 2024-11-12T18:14:48,250 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated test,ddd,1731435284929.6e290ad15dec09a0cb11782988af7c4c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:48,251 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for 6e290ad15dec09a0cb11782988af7c4c 2024-11-12T18:14:48,251 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for 6e290ad15dec09a0cb11782988af7c4c 2024-11-12T18:14:48,252 INFO [StoreOpener-6e290ad15dec09a0cb11782988af7c4c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6e290ad15dec09a0cb11782988af7c4c 2024-11-12T18:14:48,254 INFO [StoreOpener-6e290ad15dec09a0cb11782988af7c4c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6e290ad15dec09a0cb11782988af7c4c columnFamilyName family 2024-11-12T18:14:48,254 DEBUG [StoreOpener-6e290ad15dec09a0cb11782988af7c4c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:48,255 INFO [StoreOpener-6e290ad15dec09a0cb11782988af7c4c-1 {}] regionserver.HStore(327): Store=6e290ad15dec09a0cb11782988af7c4c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:48,255 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for 6e290ad15dec09a0cb11782988af7c4c 2024-11-12T18:14:48,256 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/6e290ad15dec09a0cb11782988af7c4c 2024-11-12T18:14:48,258 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/6e290ad15dec09a0cb11782988af7c4c 2024-11-12T18:14:48,258 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for 6e290ad15dec09a0cb11782988af7c4c 2024-11-12T18:14:48,258 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for 6e290ad15dec09a0cb11782988af7c4c 2024-11-12T18:14:48,261 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for 6e290ad15dec09a0cb11782988af7c4c 2024-11-12T18:14:48,262 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened 6e290ad15dec09a0cb11782988af7c4c; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66814045, jitterRate=-0.004393145442008972}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T18:14:48,263 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6e290ad15dec09a0cb11782988af7c4c 2024-11-12T18:14:48,263 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for 6e290ad15dec09a0cb11782988af7c4c: Running coprocessor pre-open hook at 1731435288251Writing region info on filesystem at 1731435288251Initializing all the Stores at 1731435288252 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435288252Cleaning up temporary data from old regions at 1731435288258 (+6 ms)Running coprocessor post-open hooks at 1731435288263 (+5 ms)Region opened successfully at 1731435288263 2024-11-12T18:14:48,264 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1731435284929.6e290ad15dec09a0cb11782988af7c4c., pid=59, masterSystemTime=1731435288244 2024-11-12T18:14:48,267 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1731435284929.6e290ad15dec09a0cb11782988af7c4c. 2024-11-12T18:14:48,267 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened test,ddd,1731435284929.6e290ad15dec09a0cb11782988af7c4c. 2024-11-12T18:14:48,268 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=6e290ad15dec09a0cb11782988af7c4c, regionState=OPEN, openSeqNum=5, regionLocation=9911683f163c,45243,1731435282573 2024-11-12T18:14:48,270 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6e290ad15dec09a0cb11782988af7c4c, server=9911683f163c,45243,1731435282573 because future has completed 2024-11-12T18:14:48,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=57 2024-11-12T18:14:48,275 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=57, state=SUCCESS, hasLock=false; OpenRegionProcedure 6e290ad15dec09a0cb11782988af7c4c, server=9911683f163c,45243,1731435282573 in 180 msec 2024-11-12T18:14:48,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6e290ad15dec09a0cb11782988af7c4c, REOPEN/MOVE in 529 msec 2024-11-12T18:14:48,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-11-12T18:14:48,351 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T18:14:48,359 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.3 set balanceSwitch=false 2024-11-12T18:14:48,375 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=375, OpenFileDescriptor=618, MaxFileDescriptor=1048576, SystemLoadAverage=799, ProcessCount=11, AvailableMemoryMB=2114 2024-11-12T18:14:48,378 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-11-12T18:14:48,379 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-11-12T18:14:48,379 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 9911683f163c,40543,1731435282322 2024-11-12T18:14:48,380 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 9911683f163c,40543,1731435282322: Simulated kill ***** 2024-11-12T18:14:48,381 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-12T18:14:48,383 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-12T18:14:48,385 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-12T18:14:48,385 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-12T18:14:48,386 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-12T18:14:48,386 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 332951232 }, "NonHeapMemoryUsage": { "committed": 153354240, "init": 7667712, "max": -1, "used": 151020408 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "9911683f163c" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "9911683f163c" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "9911683f163c" } ] } 2024-11-12T18:14:48,389 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36779 {}] master.MasterRpcServices(700): 9911683f163c,40543,1731435282322 reported a fatal error: ***** ABORTING region server 9911683f163c,40543,1731435282322: Simulated kill ***** 2024-11-12T18:14:48,391 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9911683f163c,40543,1731435282322' ***** 2024-11-12T18:14:48,391 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-12T18:14:48,392 INFO [RS:0;9911683f163c:40543 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-12T18:14:48,392 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-12T18:14:48,392 INFO [RS:0;9911683f163c:40543 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-12T18:14:48,392 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T18:14:48,392 INFO [RS:0;9911683f163c:40543 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-12T18:14:48,393 INFO [RS:0;9911683f163c:40543 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T18:14:48,393 INFO [RS:0;9911683f163c:40543 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;9911683f163c:40543. 2024-11-12T18:14:48,393 DEBUG [RS:0;9911683f163c:40543 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T18:14:48,399 DEBUG [RS:0;9911683f163c:40543 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:48,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741836_1012 (size=33749) 2024-11-12T18:14:48,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741836_1012 (size=33749) 2024-11-12T18:14:48,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741834_1010 (size=2774) 2024-11-12T18:14:48,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741834_1010 (size=2774) 2024-11-12T18:14:48,413 DEBUG [RS:0;9911683f163c:40543 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:48,413 INFO [RS:0;9911683f163c:40543 {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T18:14:48,413 INFO [RS:0;9911683f163c:40543 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40543 2024-11-12T18:14:48,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40543-0x1003533c26e0001, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9911683f163c,40543,1731435282322 2024-11-12T18:14:48,422 INFO [RS:0;9911683f163c:40543 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T18:14:48,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T18:14:48,425 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9911683f163c,40543,1731435282322] 2024-11-12T18:14:48,427 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9911683f163c,40543,1731435282322 already deleted, retry=false 2024-11-12T18:14:48,427 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 9911683f163c,40543,1731435282322 on 9911683f163c,36779,1731435281367 2024-11-12T18:14:48,432 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true 2024-11-12T18:14:48,437 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true 2024-11-12T18:14:48,437 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=60 for 9911683f163c,40543,1731435282322 (carryingMeta=true) 9911683f163c,40543,1731435282322/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@44a6fb1f[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-12T18:14:48,439 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true, isMeta: true 2024-11-12T18:14:48,442 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,40543,1731435282322-splitting 2024-11-12T18:14:48,443 INFO [PEWorker-1 {}] master.SplitWALManager(105): 9911683f163c,40543,1731435282322 WAL count=1, meta=true 2024-11-12T18:14:48,445 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta}] 2024-11-12T18:14:48,451 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:48,453 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573}] 2024-11-12T18:14:48,484 INFO [regionserver/9911683f163c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T18:14:48,494 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T18:14:48,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40543-0x1003533c26e0001, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T18:14:48,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40543-0x1003533c26e0001, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T18:14:48,526 INFO [RS:0;9911683f163c:40543 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T18:14:48,526 INFO [RS:0;9911683f163c:40543 {}] regionserver.HRegionServer(1031): Exiting; stopping=9911683f163c,40543,1731435282322; zookeeper connection closed. 2024-11-12T18:14:48,527 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@631661d9 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@631661d9 2024-11-12T18:14:48,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46767 is added to blk_1073741831_1007 (size=1321) 2024-11-12T18:14:48,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46767 is added to blk_1073741832_1008 (size=32) 2024-11-12T18:14:48,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46767 is added to blk_1073741837_1013 (size=358) 2024-11-12T18:14:48,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46767 is added to blk_1073741827_1003 (size=196) 2024-11-12T18:14:48,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=62 2024-11-12T18:14:48,645 INFO [RS_LOG_REPLAY_OPS-regionserver/9911683f163c:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(299): Splitting hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,40543,1731435282322-splitting/9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, size=33.0 K (33749bytes) 2024-11-12T18:14:48,646 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9911683f163c:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-12T18:14:48,646 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9911683f163c:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-12T18:14:48,646 INFO [RS_LOG_REPLAY_OPS-regionserver/9911683f163c:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,40543,1731435282322-splitting/9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta 2024-11-12T18:14:48,653 INFO [RS_LOG_REPLAY_OPS-regionserver/9911683f163c:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,40543,1731435282322-splitting/9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta after 5ms 2024-11-12T18:14:48,663 INFO [RS_LOG_REPLAY_OPS-regionserver/9911683f163c:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(310): Open hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,40543,1731435282322-splitting/9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta took 21ms 2024-11-12T18:14:48,673 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9911683f163c:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-12T18:14:48,681 INFO [RS_LOG_REPLAY_OPS-regionserver/9911683f163c:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta.temp 2024-11-12T18:14:48,681 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9911683f163c:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,40543,1731435282322-splitting/9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta so closing down 2024-11-12T18:14:48,681 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9911683f163c:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-12T18:14:48,684 INFO [RS_LOG_REPLAY_OPS-regionserver/9911683f163c:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta.temp 2024-11-12T18:14:48,688 INFO [RS_LOG_REPLAY_OPS-regionserver/9911683f163c:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(145): 3 split writer threads finished 2024-11-12T18:14:48,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741864_1040 (size=33508) 2024-11-12T18:14:48,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46767 is added to blk_1073741864_1040 (size=33508) 2024-11-12T18:14:48,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741864_1040 (size=33508) 2024-11-12T18:14:48,701 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta.temp (wrote 61 edits, skipped 0 edits in 3 ms) 2024-11-12T18:14:48,704 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta.temp to hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-12T18:14:48,707 INFO [RS_LOG_REPLAY_OPS-regionserver/9911683f163c:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(425): Processed 62 edits across 1 Regions in 39 ms; skipped=1; WAL=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,40543,1731435282322-splitting/9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false 2024-11-12T18:14:48,707 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9911683f163c:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(428): Completed split of hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,40543,1731435282322-splitting/9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, journal: Splitting hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,40543,1731435282322-splitting/9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, size=33.0 K (33749bytes) at 1731435288645Finishing writing output for hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,40543,1731435282322-splitting/9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta so closing down at 1731435288681 (+36 ms)Creating recovered edits writer path=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta.temp at 1731435288684 (+3 ms)3 split writer threads finished at 1731435288688 (+4 ms)Closed recovered edits writer path=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta.temp (wrote 61 edits, skipped 0 edits in 3 ms) at 1731435288701 (+13 ms)Rename recovered edits hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta.temp to hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 at 1731435288704 (+3 ms)Processed 62 edits across 1 Regions in 39 ms; skipped=1; WAL=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,40543,1731435282322-splitting/9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false at 1731435288707 (+3 ms) 2024-11-12T18:14:48,708 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9911683f163c:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,40543,1731435282322-splitting/9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta 2024-11-12T18:14:48,709 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9911683f163c:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=62 2024-11-12T18:14:48,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] master.HMaster(4169): Remote procedure done, pid=62 2024-11-12T18:14:48,719 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,40543,1731435282322-splitting/9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta to hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/oldWALs 2024-11-12T18:14:48,722 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-12T18:14:48,722 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 in 266 msec 2024-11-12T18:14:48,724 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:48,727 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-12T18:14:48,727 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 in 280 msec 2024-11-12T18:14:48,730 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,40543,1731435282322-splitting dir is empty, no logs to split. 2024-11-12T18:14:48,730 INFO [PEWorker-1 {}] master.SplitWALManager(105): 9911683f163c,40543,1731435282322 WAL count=0, meta=true 2024-11-12T18:14:48,730 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 9911683f163c,40543,1731435282322 WAL splitting is done? wals=0, meta=true 2024-11-12T18:14:48,732 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-12T18:14:48,733 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T18:14:48,734 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-12T18:14:48,877 INFO [regionserver/9911683f163c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-12T18:14:48,877 INFO [regionserver/9911683f163c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-12T18:14:48,884 DEBUG [9911683f163c:36779 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-12T18:14:48,884 DEBUG [9911683f163c:36779 {}] balancer.BalancerClusterState(204): Hosts are {9911683f163c=0} racks are {/default-rack=0} 2024-11-12T18:14:48,885 DEBUG [9911683f163c:36779 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-12T18:14:48,885 DEBUG [9911683f163c:36779 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-12T18:14:48,885 DEBUG [9911683f163c:36779 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-12T18:14:48,885 DEBUG [9911683f163c:36779 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-12T18:14:48,885 INFO [9911683f163c:36779 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-12T18:14:48,885 INFO [9911683f163c:36779 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-12T18:14:48,885 DEBUG [9911683f163c:36779 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-12T18:14:48,885 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9911683f163c,34847,1731435282512 2024-11-12T18:14:48,886 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9911683f163c,34847,1731435282512, state=OPENING 2024-11-12T18:14:48,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:48,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:48,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:48,889 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:48,889 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T18:14:48,889 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:48,889 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:48,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,34847,1731435282512}] 2024-11-12T18:14:49,047 INFO [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-12T18:14:49,047 INFO [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T18:14:49,047 INFO [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-12T18:14:49,049 INFO [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9911683f163c%2C34847%2C1731435282512.meta, suffix=.meta, logDir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,34847,1731435282512, archiveDir=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/oldWALs, maxLogs=32 2024-11-12T18:14:49,066 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,34847,1731435282512/9911683f163c%2C34847%2C1731435282512.meta.1731435289051.meta, exclude list is [], retry=0 2024-11-12T18:14:49,070 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46767,DS-12f17fcf-9e85-422d-9538-7fd753218ef4,DISK] 2024-11-12T18:14:49,070 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35529,DS-c1ead257-8ceb-4707-bb84-0752011c7478,DISK] 2024-11-12T18:14:49,070 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42977,DS-daada605-434c-4a56-9ec7-9e1ddc6c103c,DISK] 2024-11-12T18:14:49,073 INFO [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,34847,1731435282512/9911683f163c%2C34847%2C1731435282512.meta.1731435289051.meta 2024-11-12T18:14:49,073 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35205:35205),(127.0.0.1/127.0.0.1:36981:36981),(127.0.0.1/127.0.0.1:33383:33383)] 2024-11-12T18:14:49,074 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-12T18:14:49,074 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-12T18:14:49,074 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-12T18:14:49,074 INFO [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-12T18:14:49,074 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-12T18:14:49,074 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:49,075 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-12T18:14:49,075 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-12T18:14:49,076 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-12T18:14:49,078 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T18:14:49,078 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:49,079 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T18:14:49,079 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-12T18:14:49,080 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T18:14:49,080 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:49,080 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T18:14:49,081 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-12T18:14:49,082 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T18:14:49,082 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:49,082 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T18:14:49,082 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-12T18:14:49,083 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T18:14:49,083 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:49,084 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T18:14:49,084 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T18:14:49,084 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/hbase/meta/1588230740 2024-11-12T18:14:49,087 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/hbase/meta/1588230740 2024-11-12T18:14:49,087 INFO [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-12T18:14:49,102 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5793): Applied 277, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=64, path=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-12T18:14:49,106 INFO [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=29.41 KB heapSize=50.67 KB 2024-11-12T18:14:49,182 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/hbase/meta/1588230740/.tmp/info/1468f17511d44aea85de6ba622316827 is 138, key is test,bbb,1731435284929.2248a0d1296d1dbe4f8c9805798db03c./info:regioninfo/1731435286571/Put/seqid=0 2024-11-12T18:14:49,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46767 is added to blk_1073741866_1042 (size=36385) 2024-11-12T18:14:49,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741866_1042 (size=36385) 2024-11-12T18:14:49,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741866_1042 (size=36385) 2024-11-12T18:14:49,197 INFO [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.26 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/hbase/meta/1588230740/.tmp/info/1468f17511d44aea85de6ba622316827 2024-11-12T18:14:49,271 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/hbase/meta/1588230740/.tmp/ns/70ee187e0ee14f679d0f6a7244480265 is 43, key is default/ns:d/1731435284686/Put/seqid=0 2024-11-12T18:14:49,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741867_1043 (size=5153) 2024-11-12T18:14:49,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46767 is added to blk_1073741867_1043 (size=5153) 2024-11-12T18:14:49,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741867_1043 (size=5153) 2024-11-12T18:14:49,284 INFO [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/hbase/meta/1588230740/.tmp/ns/70ee187e0ee14f679d0f6a7244480265 2024-11-12T18:14:49,309 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/hbase/meta/1588230740/.tmp/table/7eb1fd79046e44c4a75923a17c3e6842 is 40, key is test/table:state/1731435286593/Put/seqid=0 2024-11-12T18:14:49,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741868_1044 (size=5165) 2024-11-12T18:14:49,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46767 is added to blk_1073741868_1044 (size=5165) 2024-11-12T18:14:49,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741868_1044 (size=5165) 2024-11-12T18:14:49,320 INFO [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/hbase/meta/1588230740/.tmp/table/7eb1fd79046e44c4a75923a17c3e6842 2024-11-12T18:14:49,330 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/hbase/meta/1588230740/.tmp/info/1468f17511d44aea85de6ba622316827 as hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/hbase/meta/1588230740/info/1468f17511d44aea85de6ba622316827 2024-11-12T18:14:49,340 INFO [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/hbase/meta/1588230740/info/1468f17511d44aea85de6ba622316827, entries=273, sequenceid=64, filesize=35.5 K 2024-11-12T18:14:49,343 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/hbase/meta/1588230740/.tmp/ns/70ee187e0ee14f679d0f6a7244480265 as hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/hbase/meta/1588230740/ns/70ee187e0ee14f679d0f6a7244480265 2024-11-12T18:14:49,352 INFO [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/hbase/meta/1588230740/ns/70ee187e0ee14f679d0f6a7244480265, entries=2, sequenceid=64, filesize=5.0 K 2024-11-12T18:14:49,354 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/hbase/meta/1588230740/.tmp/table/7eb1fd79046e44c4a75923a17c3e6842 as hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/hbase/meta/1588230740/table/7eb1fd79046e44c4a75923a17c3e6842 2024-11-12T18:14:49,362 INFO [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/hbase/meta/1588230740/table/7eb1fd79046e44c4a75923a17c3e6842, entries=2, sequenceid=64, filesize=5.0 K 2024-11-12T18:14:49,368 INFO [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(3140): Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 260ms, sequenceid=64, compaction requested=false; wal=null 2024-11-12T18:14:49,369 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-12T18:14:49,373 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-12T18:14:49,375 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T18:14:49,375 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T18:14:49,376 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T18:14:49,378 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T18:14:49,380 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/hbase/meta/1588230740/recovered.edits/64.seqid, newMaxSeqId=64, maxSeqId=1 2024-11-12T18:14:49,382 INFO [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=65; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63485880, jitterRate=-0.05398666858673096}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T18:14:49,382 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-12T18:14:49,382 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731435289075Writing region info on filesystem at 1731435289075Initializing all the Stores at 1731435289076 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731435289076Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731435289076Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435289076Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731435289076Obtaining lock to block concurrent updates at 1731435289106 (+30 ms)Preparing flush snapshotting stores in 1588230740 at 1731435289106Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=30111, getHeapSize=51824, getOffHeapSize=0, getCellsCount=277 at 1731435289108 (+2 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731435289109 (+1 ms)Flushing 1588230740/info: creating writer at 1731435289110 (+1 ms)Flushing 1588230740/info: appending metadata at 1731435289169 (+59 ms)Flushing 1588230740/info: closing flushed file at 1731435289173 (+4 ms)Flushing 1588230740/ns: creating writer at 1731435289251 (+78 ms)Flushing 1588230740/ns: appending metadata at 1731435289269 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1731435289270 (+1 ms)Flushing 1588230740/table: creating writer at 1731435289293 (+23 ms)Flushing 1588230740/table: appending metadata at 1731435289309 (+16 ms)Flushing 1588230740/table: closing flushed file at 1731435289309Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7005a5bc: reopening flushed file at 1731435289329 (+20 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@204e634: reopening flushed file at 1731435289342 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ba549e: reopening flushed file at 1731435289353 (+11 ms)Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 260ms, sequenceid=64, compaction requested=false; wal=null at 1731435289368 (+15 ms)Cleaning up temporary data from old regions at 1731435289375 (+7 ms)Running coprocessor post-open hooks at 1731435289382 (+7 ms)Region opened successfully at 1731435289382 2024-11-12T18:14:49,384 INFO [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=64, masterSystemTime=1731435289041 2024-11-12T18:14:49,387 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-12T18:14:49,387 INFO [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-12T18:14:49,388 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=65, regionLocation=9911683f163c,34847,1731435282512 2024-11-12T18:14:49,388 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9911683f163c,34847,1731435282512, state=OPEN 2024-11-12T18:14:49,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:49,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:49,391 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:49,391 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:49,391 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=64, ppid=63, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9911683f163c,34847,1731435282512 2024-11-12T18:14:49,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:49,391 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:49,394 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-12T18:14:49,394 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,34847,1731435282512 in 502 msec 2024-11-12T18:14:49,396 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=60 2024-11-12T18:14:49,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 662 msec 2024-11-12T18:14:49,397 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 9911683f163c,40543,1731435282322 had 9 regions 2024-11-12T18:14:49,399 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true, isMeta: false 2024-11-12T18:14:49,401 INFO [PEWorker-1 {}] master.SplitWALManager(105): 9911683f163c,40543,1731435282322 WAL count=1, meta=false 2024-11-12T18:14:49,401 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060}] 2024-11-12T18:14:49,403 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=9911683f163c,34847,1731435282512 2024-11-12T18:14:49,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512}] 2024-11-12T18:14:49,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=66 2024-11-12T18:14:49,578 INFO [RS_LOG_REPLAY_OPS-regionserver/9911683f163c:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(299): Splitting hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,40543,1731435282322-splitting/9911683f163c%2C40543%2C1731435282322.1731435284060, size=2.7 K (2774bytes) 2024-11-12T18:14:49,578 INFO [RS_LOG_REPLAY_OPS-regionserver/9911683f163c:0-0 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,40543,1731435282322-splitting/9911683f163c%2C40543%2C1731435282322.1731435284060 2024-11-12T18:14:49,579 INFO [RS_LOG_REPLAY_OPS-regionserver/9911683f163c:0-0 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,40543,1731435282322-splitting/9911683f163c%2C40543%2C1731435282322.1731435284060 after 1ms 2024-11-12T18:14:49,582 INFO [RS_LOG_REPLAY_OPS-regionserver/9911683f163c:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(310): Open hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,40543,1731435282322-splitting/9911683f163c%2C40543%2C1731435282322.1731435284060 took 4ms 2024-11-12T18:14:49,586 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9911683f163c:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 1b81a9be48fbe50dde523592a87a4611: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-12T18:14:49,588 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9911683f163c:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 56e168ba1a3103978bee5b6c43feba36: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-12T18:14:49,591 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9911683f163c:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 2248a0d1296d1dbe4f8c9805798db03c: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-12T18:14:49,593 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9911683f163c:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for fde2b7fbe9fa7c8a87ab544f5542d82a: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-12T18:14:49,595 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9911683f163c:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 4b703e982cd6bbf72ec2f771b120c649: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-12T18:14:49,598 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9911683f163c:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 9fbe24cda440bbf727cd581ab33ad7a9: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-12T18:14:49,600 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9911683f163c:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for d59e820eb0eef2601f5990d32a8adbc9: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-12T18:14:49,601 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9911683f163c:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 6e290ad15dec09a0cb11782988af7c4c: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-12T18:14:49,603 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9911683f163c:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 26fc65f8daf71d5831131b1bc6746a63: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-12T18:14:49,603 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9911683f163c:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,40543,1731435282322-splitting/9911683f163c%2C40543%2C1731435282322.1731435284060 so closing down 2024-11-12T18:14:49,603 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9911683f163c:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-12T18:14:49,603 INFO [RS_LOG_REPLAY_OPS-regionserver/9911683f163c:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(145): 3 split writer threads finished 2024-11-12T18:14:49,604 INFO [RS_LOG_REPLAY_OPS-regionserver/9911683f163c:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(425): Processed 10 edits across 0 Regions in 20 ms; skipped=10; WAL=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,40543,1731435282322-splitting/9911683f163c%2C40543%2C1731435282322.1731435284060, size=2.7 K, length=2774, corrupted=false, cancelled=false 2024-11-12T18:14:49,604 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9911683f163c:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(428): Completed split of hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,40543,1731435282322-splitting/9911683f163c%2C40543%2C1731435282322.1731435284060, journal: Splitting hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,40543,1731435282322-splitting/9911683f163c%2C40543%2C1731435282322.1731435284060, size=2.7 K (2774bytes) at 1731435289578Finishing writing output for hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,40543,1731435282322-splitting/9911683f163c%2C40543%2C1731435282322.1731435284060 so closing down at 1731435289603 (+25 ms)3 split writer threads finished at 1731435289603Processed 10 edits across 0 Regions in 20 ms; skipped=10; WAL=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,40543,1731435282322-splitting/9911683f163c%2C40543%2C1731435282322.1731435284060, size=2.7 K, length=2774, corrupted=false, cancelled=false at 1731435289604 (+1 ms) 2024-11-12T18:14:49,604 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9911683f163c:0-0 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,40543,1731435282322-splitting/9911683f163c%2C40543%2C1731435282322.1731435284060 2024-11-12T18:14:49,604 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9911683f163c:0-0 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=66 2024-11-12T18:14:49,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36779 {}] master.HMaster(4169): Remote procedure done, pid=66 2024-11-12T18:14:49,609 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,40543,1731435282322-splitting/9911683f163c%2C40543%2C1731435282322.1731435284060 to hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/oldWALs 2024-11-12T18:14:49,611 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-12T18:14:49,612 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 in 205 msec 2024-11-12T18:14:49,613 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=9911683f163c,34847,1731435282512 2024-11-12T18:14:49,618 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=60 2024-11-12T18:14:49,618 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 in 212 msec 2024-11-12T18:14:49,620 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,40543,1731435282322-splitting dir is empty, no logs to split. 2024-11-12T18:14:49,620 INFO [PEWorker-1 {}] master.SplitWALManager(105): 9911683f163c,40543,1731435282322 WAL count=0, meta=false 2024-11-12T18:14:49,620 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 9911683f163c,40543,1731435282322 WAL splitting is done? wals=0, meta=false 2024-11-12T18:14:49,623 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN}, {pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN}, {pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN}, {pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN}, {pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN}, {pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN}, {pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN}, {pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN}] 2024-11-12T18:14:49,624 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN 2024-11-12T18:14:49,625 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-12T18:14:49,626 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN 2024-11-12T18:14:49,627 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-12T18:14:49,628 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN 2024-11-12T18:14:49,629 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-12T18:14:49,630 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN 2024-11-12T18:14:49,631 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-12T18:14:49,633 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN 2024-11-12T18:14:49,633 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-12T18:14:49,634 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN 2024-11-12T18:14:49,635 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-12T18:14:49,636 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN 2024-11-12T18:14:49,637 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-12T18:14:49,638 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN 2024-11-12T18:14:49,638 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-12T18:14:49,776 DEBUG [9911683f163c:36779 {}] balancer.BalancerClusterState(204): Hosts are {9911683f163c=0} racks are {/default-rack=0} 2024-11-12T18:14:49,776 DEBUG [9911683f163c:36779 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-12T18:14:49,776 DEBUG [9911683f163c:36779 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-12T18:14:49,776 DEBUG [9911683f163c:36779 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-12T18:14:49,776 DEBUG [9911683f163c:36779 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-12T18:14:49,776 INFO [9911683f163c:36779 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-12T18:14:49,776 INFO [9911683f163c:36779 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-12T18:14:49,776 DEBUG [9911683f163c:36779 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-12T18:14:49,777 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=fde2b7fbe9fa7c8a87ab544f5542d82a, regionState=OPENING, regionLocation=9911683f163c,45243,1731435282573 2024-11-12T18:14:49,778 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=1b81a9be48fbe50dde523592a87a4611, regionState=OPENING, regionLocation=9911683f163c,45243,1731435282573 2024-11-12T18:14:49,779 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=4b703e982cd6bbf72ec2f771b120c649, regionState=OPENING, regionLocation=9911683f163c,45243,1731435282573 2024-11-12T18:14:49,779 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 9911683f163c:40543 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 9911683f163c/172.17.0.3:40543 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:49,779 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=26fc65f8daf71d5831131b1bc6746a63, regionState=OPENING, regionLocation=9911683f163c,45243,1731435282573 2024-11-12T18:14:49,780 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=56e168ba1a3103978bee5b6c43feba36, regionState=OPENING, regionLocation=9911683f163c,34847,1731435282512 2024-11-12T18:14:49,780 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=9fbe24cda440bbf727cd581ab33ad7a9, regionState=OPENING, regionLocation=9911683f163c,34847,1731435282512 2024-11-12T18:14:49,781 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=d59e820eb0eef2601f5990d32a8adbc9, regionState=OPENING, regionLocation=9911683f163c,34847,1731435282512 2024-11-12T18:14:49,781 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=9911683f163c,40543,1731435282322, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=9911683f163c,40543,1731435282322, seqNum=-1, error=java.net.ConnectException: Call to address=9911683f163c:40543 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 9911683f163c/172.17.0.3:40543 2024-11-12T18:14:49,781 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=73 updating hbase:meta row=2248a0d1296d1dbe4f8c9805798db03c, regionState=OPENING, regionLocation=9911683f163c,34847,1731435282512 2024-11-12T18:14:49,782 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=9911683f163c,40543,1731435282322, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-12T18:14:49,782 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=9911683f163c,40543,1731435282322, seqNum=-1 from cache 2024-11-12T18:14:49,783 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 9911683f163c:40543 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 9911683f163c/172.17.0.3:40543 2024-11-12T18:14:49,784 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 9911683f163c:40543 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 9911683f163c/172.17.0.3:40543 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:49,785 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=9911683f163c,40543,1731435282322, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=9911683f163c:40543 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 9911683f163c/172.17.0.3:40543 2024-11-12T18:14:49,785 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=9911683f163c,40543,1731435282322, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=9911683f163c:40543 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 9911683f163c/172.17.0.3:40543 2024-11-12T18:14:49,785 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=9911683f163c,40543,1731435282322, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=9911683f163c:40543 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 9911683f163c/172.17.0.3:40543 2024-11-12T18:14:49,786 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=9911683f163c,40543,1731435282322, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=9911683f163c:40543 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 9911683f163c/172.17.0.3:40543 2024-11-12T18:14:49,786 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=9911683f163c,40543,1731435282322, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=9911683f163c:40543 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 9911683f163c/172.17.0.3:40543 2024-11-12T18:14:49,786 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=9911683f163c,40543,1731435282322, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=9911683f163c:40543 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 9911683f163c/172.17.0.3:40543 2024-11-12T18:14:49,786 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=9911683f163c,40543,1731435282322, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=9911683f163c:40543 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 9911683f163c/172.17.0.3:40543 2024-11-12T18:14:49,787 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 9911683f163c:40543 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 9911683f163c/172.17.0.3:40543 2024-11-12T18:14:49,886 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:14:49,886 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,34847,1731435282512, seqNum=-1] 2024-11-12T18:14:49,886 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:14:49,888 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55895, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:14:49,890 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN because future has completed 2024-11-12T18:14:49,891 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure fde2b7fbe9fa7c8a87ab544f5542d82a, server=9911683f163c,45243,1731435282573}] 2024-11-12T18:14:49,898 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN because future has completed 2024-11-12T18:14:49,898 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure d59e820eb0eef2601f5990d32a8adbc9, server=9911683f163c,34847,1731435282512}] 2024-11-12T18:14:49,899 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN because future has completed 2024-11-12T18:14:49,900 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN because future has completed 2024-11-12T18:14:49,902 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN because future has completed 2024-11-12T18:14:49,903 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN because future has completed 2024-11-12T18:14:49,903 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26fc65f8daf71d5831131b1bc6746a63, server=9911683f163c,45243,1731435282573}] 2024-11-12T18:14:49,904 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN because future has completed 2024-11-12T18:14:49,905 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN because future has completed 2024-11-12T18:14:49,907 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9fbe24cda440bbf727cd581ab33ad7a9, server=9911683f163c,34847,1731435282512}] 2024-11-12T18:14:49,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 56e168ba1a3103978bee5b6c43feba36, server=9911683f163c,34847,1731435282512}] 2024-11-12T18:14:49,913 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573}] 2024-11-12T18:14:49,916 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b81a9be48fbe50dde523592a87a4611, server=9911683f163c,45243,1731435282573}] 2024-11-12T18:14:49,919 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2248a0d1296d1dbe4f8c9805798db03c, server=9911683f163c,34847,1731435282512}] 2024-11-12T18:14:50,048 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(132): Open test,nnn,1731435284929.4b703e982cd6bbf72ec2f771b120c649. 2024-11-12T18:14:50,048 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7752): Opening region: {ENCODED => 4b703e982cd6bbf72ec2f771b120c649, NAME => 'test,nnn,1731435284929.4b703e982cd6bbf72ec2f771b120c649.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-12T18:14:50,048 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4b703e982cd6bbf72ec2f771b120c649 2024-11-12T18:14:50,048 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(898): Instantiated test,nnn,1731435284929.4b703e982cd6bbf72ec2f771b120c649.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:50,048 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7794): checking encryption for 4b703e982cd6bbf72ec2f771b120c649 2024-11-12T18:14:50,048 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7797): checking classloading for 4b703e982cd6bbf72ec2f771b120c649 2024-11-12T18:14:50,050 INFO [StoreOpener-4b703e982cd6bbf72ec2f771b120c649-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4b703e982cd6bbf72ec2f771b120c649 2024-11-12T18:14:50,051 INFO [StoreOpener-4b703e982cd6bbf72ec2f771b120c649-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b703e982cd6bbf72ec2f771b120c649 columnFamilyName family 2024-11-12T18:14:50,051 DEBUG [StoreOpener-4b703e982cd6bbf72ec2f771b120c649-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:50,052 INFO [StoreOpener-4b703e982cd6bbf72ec2f771b120c649-1 {}] regionserver.HStore(327): Store=4b703e982cd6bbf72ec2f771b120c649/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:50,052 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1038): replaying wal for 4b703e982cd6bbf72ec2f771b120c649 2024-11-12T18:14:50,053 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/4b703e982cd6bbf72ec2f771b120c649 2024-11-12T18:14:50,054 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/4b703e982cd6bbf72ec2f771b120c649 2024-11-12T18:14:50,055 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1048): stopping wal replay for 4b703e982cd6bbf72ec2f771b120c649 2024-11-12T18:14:50,055 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1060): Cleaning up temporary data for 4b703e982cd6bbf72ec2f771b120c649 2024-11-12T18:14:50,055 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,kkk,1731435284929.d59e820eb0eef2601f5990d32a8adbc9. 2024-11-12T18:14:50,055 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => d59e820eb0eef2601f5990d32a8adbc9, NAME => 'test,kkk,1731435284929.d59e820eb0eef2601f5990d32a8adbc9.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-12T18:14:50,056 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d59e820eb0eef2601f5990d32a8adbc9 2024-11-12T18:14:50,056 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,kkk,1731435284929.d59e820eb0eef2601f5990d32a8adbc9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:50,056 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for d59e820eb0eef2601f5990d32a8adbc9 2024-11-12T18:14:50,056 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for d59e820eb0eef2601f5990d32a8adbc9 2024-11-12T18:14:50,058 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1093): writing seq id for 4b703e982cd6bbf72ec2f771b120c649 2024-11-12T18:14:50,058 INFO [StoreOpener-d59e820eb0eef2601f5990d32a8adbc9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d59e820eb0eef2601f5990d32a8adbc9 2024-11-12T18:14:50,059 INFO [StoreOpener-d59e820eb0eef2601f5990d32a8adbc9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d59e820eb0eef2601f5990d32a8adbc9 columnFamilyName family 2024-11-12T18:14:50,059 DEBUG [StoreOpener-d59e820eb0eef2601f5990d32a8adbc9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:50,059 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1114): Opened 4b703e982cd6bbf72ec2f771b120c649; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73722552, jitterRate=0.09855163097381592}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T18:14:50,059 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4b703e982cd6bbf72ec2f771b120c649 2024-11-12T18:14:50,059 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1006): Region open journal for 4b703e982cd6bbf72ec2f771b120c649: Running coprocessor pre-open hook at 1731435290049Writing region info on filesystem at 1731435290049Initializing all the Stores at 1731435290050 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435290050Cleaning up temporary data from old regions at 1731435290055 (+5 ms)Running coprocessor post-open hooks at 1731435290059 (+4 ms)Region opened successfully at 1731435290059 2024-11-12T18:14:50,059 INFO [StoreOpener-d59e820eb0eef2601f5990d32a8adbc9-1 {}] regionserver.HStore(327): Store=d59e820eb0eef2601f5990d32a8adbc9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:50,060 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for d59e820eb0eef2601f5990d32a8adbc9 2024-11-12T18:14:50,060 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1731435284929.4b703e982cd6bbf72ec2f771b120c649., pid=80, masterSystemTime=1731435290043 2024-11-12T18:14:50,061 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/d59e820eb0eef2601f5990d32a8adbc9 2024-11-12T18:14:50,063 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/d59e820eb0eef2601f5990d32a8adbc9 2024-11-12T18:14:50,063 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1731435284929.4b703e982cd6bbf72ec2f771b120c649. 2024-11-12T18:14:50,063 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(153): Opened test,nnn,1731435284929.4b703e982cd6bbf72ec2f771b120c649. 2024-11-12T18:14:50,064 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,yyy,1731435284929.fde2b7fbe9fa7c8a87ab544f5542d82a. 2024-11-12T18:14:50,064 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => fde2b7fbe9fa7c8a87ab544f5542d82a, NAME => 'test,yyy,1731435284929.fde2b7fbe9fa7c8a87ab544f5542d82a.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-12T18:14:50,064 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test fde2b7fbe9fa7c8a87ab544f5542d82a 2024-11-12T18:14:50,064 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,yyy,1731435284929.fde2b7fbe9fa7c8a87ab544f5542d82a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:50,064 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for fde2b7fbe9fa7c8a87ab544f5542d82a 2024-11-12T18:14:50,064 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for fde2b7fbe9fa7c8a87ab544f5542d82a 2024-11-12T18:14:50,065 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for d59e820eb0eef2601f5990d32a8adbc9 2024-11-12T18:14:50,065 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for d59e820eb0eef2601f5990d32a8adbc9 2024-11-12T18:14:50,066 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-11-12T18:14:50,066 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,067 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for d59e820eb0eef2601f5990d32a8adbc9 2024-11-12T18:14:50,066 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,068 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened d59e820eb0eef2601f5990d32a8adbc9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61619855, jitterRate=-0.08179260790348053}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T18:14:50,068 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d59e820eb0eef2601f5990d32a8adbc9 2024-11-12T18:14:50,068 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for d59e820eb0eef2601f5990d32a8adbc9: Running coprocessor pre-open hook at 1731435290056Writing region info on filesystem at 1731435290056Initializing all the Stores at 1731435290057 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435290057Cleaning up temporary data from old regions at 1731435290065 (+8 ms)Running coprocessor post-open hooks at 1731435290068 (+3 ms)Region opened successfully at 1731435290068 2024-11-12T18:14:50,069 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1731435284929.d59e820eb0eef2601f5990d32a8adbc9., pid=76, masterSystemTime=1731435290051 2024-11-12T18:14:50,072 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1731435284929.d59e820eb0eef2601f5990d32a8adbc9. 2024-11-12T18:14:50,072 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened test,kkk,1731435284929.d59e820eb0eef2601f5990d32a8adbc9. 2024-11-12T18:14:50,073 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,ppp,1731435284929.9fbe24cda440bbf727cd581ab33ad7a9. 2024-11-12T18:14:50,073 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => 9fbe24cda440bbf727cd581ab33ad7a9, NAME => 'test,ppp,1731435284929.9fbe24cda440bbf727cd581ab33ad7a9.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-12T18:14:50,073 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 9fbe24cda440bbf727cd581ab33ad7a9 2024-11-12T18:14:50,073 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,ppp,1731435284929.9fbe24cda440bbf727cd581ab33ad7a9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:50,073 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for 9fbe24cda440bbf727cd581ab33ad7a9 2024-11-12T18:14:50,074 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for 9fbe24cda440bbf727cd581ab33ad7a9 2024-11-12T18:14:50,074 INFO [StoreOpener-fde2b7fbe9fa7c8a87ab544f5542d82a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region fde2b7fbe9fa7c8a87ab544f5542d82a 2024-11-12T18:14:50,075 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T18:14:50,079 INFO [StoreOpener-fde2b7fbe9fa7c8a87ab544f5542d82a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fde2b7fbe9fa7c8a87ab544f5542d82a columnFamilyName family 2024-11-12T18:14:50,079 DEBUG [StoreOpener-fde2b7fbe9fa7c8a87ab544f5542d82a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:50,079 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 4 more 2024-11-12T18:14:50,080 INFO [StoreOpener-fde2b7fbe9fa7c8a87ab544f5542d82a-1 {}] regionserver.HStore(327): Store=fde2b7fbe9fa7c8a87ab544f5542d82a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:50,081 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T18:14:50,081 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=82, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2248a0d1296d1dbe4f8c9805798db03c, server=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,081 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for fde2b7fbe9fa7c8a87ab544f5542d82a 2024-11-12T18:14:50,081 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,081 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=82, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2248a0d1296d1dbe4f8c9805798db03c, server=9911683f163c,34847,1731435282512 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,082 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/fde2b7fbe9fa7c8a87ab544f5542d82a 2024-11-12T18:14:50,083 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/fde2b7fbe9fa7c8a87ab544f5542d82a 2024-11-12T18:14:50,084 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for fde2b7fbe9fa7c8a87ab544f5542d82a 2024-11-12T18:14:50,084 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for fde2b7fbe9fa7c8a87ab544f5542d82a 2024-11-12T18:14:50,087 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for fde2b7fbe9fa7c8a87ab544f5542d82a 2024-11-12T18:14:50,088 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened fde2b7fbe9fa7c8a87ab544f5542d82a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60921256, jitterRate=-0.09220254421234131}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T18:14:50,088 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fde2b7fbe9fa7c8a87ab544f5542d82a 2024-11-12T18:14:50,088 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for fde2b7fbe9fa7c8a87ab544f5542d82a: Running coprocessor pre-open hook at 1731435290065Writing region info on filesystem at 1731435290065Initializing all the Stores at 1731435290073 (+8 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435290073Cleaning up temporary data from old regions at 1731435290084 (+11 ms)Running coprocessor post-open hooks at 1731435290088 (+4 ms)Region opened successfully at 1731435290088 2024-11-12T18:14:50,089 INFO [StoreOpener-9fbe24cda440bbf727cd581ab33ad7a9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 9fbe24cda440bbf727cd581ab33ad7a9 2024-11-12T18:14:50,090 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1731435284929.fde2b7fbe9fa7c8a87ab544f5542d82a., pid=75, masterSystemTime=1731435290043 2024-11-12T18:14:50,090 INFO [StoreOpener-9fbe24cda440bbf727cd581ab33ad7a9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9fbe24cda440bbf727cd581ab33ad7a9 columnFamilyName family 2024-11-12T18:14:50,090 DEBUG [StoreOpener-9fbe24cda440bbf727cd581ab33ad7a9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:50,091 INFO [StoreOpener-9fbe24cda440bbf727cd581ab33ad7a9-1 {}] regionserver.HStore(327): Store=9fbe24cda440bbf727cd581ab33ad7a9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:50,091 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for 9fbe24cda440bbf727cd581ab33ad7a9 2024-11-12T18:14:50,093 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/9fbe24cda440bbf727cd581ab33ad7a9 2024-11-12T18:14:50,095 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1731435284929.fde2b7fbe9fa7c8a87ab544f5542d82a. 2024-11-12T18:14:50,095 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,yyy,1731435284929.fde2b7fbe9fa7c8a87ab544f5542d82a. 2024-11-12T18:14:50,095 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,ggg,1731435284929.26fc65f8daf71d5831131b1bc6746a63. 2024-11-12T18:14:50,095 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => 26fc65f8daf71d5831131b1bc6746a63, NAME => 'test,ggg,1731435284929.26fc65f8daf71d5831131b1bc6746a63.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-12T18:14:50,096 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 26fc65f8daf71d5831131b1bc6746a63 2024-11-12T18:14:50,096 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,ggg,1731435284929.26fc65f8daf71d5831131b1bc6746a63.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:50,096 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for 26fc65f8daf71d5831131b1bc6746a63 2024-11-12T18:14:50,096 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for 26fc65f8daf71d5831131b1bc6746a63 2024-11-12T18:14:50,097 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/9fbe24cda440bbf727cd581ab33ad7a9 2024-11-12T18:14:50,098 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for 9fbe24cda440bbf727cd581ab33ad7a9 2024-11-12T18:14:50,098 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for 9fbe24cda440bbf727cd581ab33ad7a9 2024-11-12T18:14:50,098 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T18:14:50,098 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,098 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T18:14:50,099 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T18:14:50,101 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T18:14:50,101 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T18:14:50,101 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T18:14:50,101 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T18:14:50,103 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for 9fbe24cda440bbf727cd581ab33ad7a9 2024-11-12T18:14:50,104 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened 9fbe24cda440bbf727cd581ab33ad7a9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73289094, jitterRate=0.0920926034450531}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T18:14:50,104 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9fbe24cda440bbf727cd581ab33ad7a9 2024-11-12T18:14:50,104 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for 9fbe24cda440bbf727cd581ab33ad7a9: Running coprocessor pre-open hook at 1731435290074Writing region info on filesystem at 1731435290074Initializing all the Stores at 1731435290076 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435290077 (+1 ms)Cleaning up temporary data from old regions at 1731435290098 (+21 ms)Running coprocessor post-open hooks at 1731435290104 (+6 ms)Region opened successfully at 1731435290104 2024-11-12T18:14:50,104 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-11-12T18:14:50,106 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1731435284929.9fbe24cda440bbf727cd581ab33ad7a9., pid=78, masterSystemTime=1731435290051 2024-11-12T18:14:50,107 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36779 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,110 INFO [StoreOpener-26fc65f8daf71d5831131b1bc6746a63-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 26fc65f8daf71d5831131b1bc6746a63 2024-11-12T18:14:50,114 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T18:14:50,114 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T18:14:50,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T18:14:50,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=82, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T18:14:50,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T18:14:50,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,116 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T18:14:50,117 INFO [StoreOpener-26fc65f8daf71d5831131b1bc6746a63-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 26fc65f8daf71d5831131b1bc6746a63 columnFamilyName family 2024-11-12T18:14:50,117 DEBUG [StoreOpener-26fc65f8daf71d5831131b1bc6746a63-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:50,118 INFO [StoreOpener-26fc65f8daf71d5831131b1bc6746a63-1 {}] regionserver.HStore(327): Store=26fc65f8daf71d5831131b1bc6746a63/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:50,118 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for 26fc65f8daf71d5831131b1bc6746a63 2024-11-12T18:14:50,119 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/26fc65f8daf71d5831131b1bc6746a63 2024-11-12T18:14:50,120 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T18:14:50,121 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6e290ad15dec09a0cb11782988af7c4c, REOPEN/MOVE 2024-11-12T18:14:50,121 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/26fc65f8daf71d5831131b1bc6746a63 2024-11-12T18:14:50,122 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for 26fc65f8daf71d5831131b1bc6746a63 2024-11-12T18:14:50,122 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for 26fc65f8daf71d5831131b1bc6746a63 2024-11-12T18:14:50,125 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for 26fc65f8daf71d5831131b1bc6746a63 2024-11-12T18:14:50,126 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened 26fc65f8daf71d5831131b1bc6746a63; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65849565, jitterRate=-0.01876501739025116}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T18:14:50,126 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 26fc65f8daf71d5831131b1bc6746a63 2024-11-12T18:14:50,126 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for 26fc65f8daf71d5831131b1bc6746a63: Running coprocessor pre-open hook at 1731435290096Writing region info on filesystem at 1731435290096Initializing all the Stores at 1731435290097 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435290097Cleaning up temporary data from old regions at 1731435290122 (+25 ms)Running coprocessor post-open hooks at 1731435290126 (+4 ms)Region opened successfully at 1731435290126 2024-11-12T18:14:50,127 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1731435284929.26fc65f8daf71d5831131b1bc6746a63., pid=77, masterSystemTime=1731435290043 2024-11-12T18:14:50,128 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36779 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,131 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true 2024-11-12T18:14:50,131 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T18:14:50,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN 2024-11-12T18:14:50,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN 2024-11-12T18:14:50,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN 2024-11-12T18:14:50,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN 2024-11-12T18:14:50,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN 2024-11-12T18:14:50,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN 2024-11-12T18:14:50,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN 2024-11-12T18:14:50,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN 2024-11-12T18:14:50,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure fde2b7fbe9fa7c8a87ab544f5542d82a, server=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure d59e820eb0eef2601f5990d32a8adbc9, server=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,133 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26fc65f8daf71d5831131b1bc6746a63, server=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,133 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9fbe24cda440bbf727cd581ab33ad7a9, server=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,133 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 56e168ba1a3103978bee5b6c43feba36, server=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,133 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=70, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,133 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b81a9be48fbe50dde523592a87a4611, server=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,133 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=82, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2248a0d1296d1dbe4f8c9805798db03c, server=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,133 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,133 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,133 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,133 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,133 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,134 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,134 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,134 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,134 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN 2024-11-12T18:14:50,134 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,134 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN 2024-11-12T18:14:50,134 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,134 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN 2024-11-12T18:14:50,134 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,134 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN 2024-11-12T18:14:50,134 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,134 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN 2024-11-12T18:14:50,134 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,134 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN 2024-11-12T18:14:50,134 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,134 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN 2024-11-12T18:14:50,134 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,134 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN 2024-11-12T18:14:50,134 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure fde2b7fbe9fa7c8a87ab544f5542d82a, server=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,134 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure d59e820eb0eef2601f5990d32a8adbc9, server=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,134 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26fc65f8daf71d5831131b1bc6746a63, server=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,134 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9fbe24cda440bbf727cd581ab33ad7a9, server=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,134 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 56e168ba1a3103978bee5b6c43feba36, server=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,134 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=70, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,134 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b81a9be48fbe50dde523592a87a4611, server=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,134 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=82, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2248a0d1296d1dbe4f8c9805798db03c, server=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,135 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 30 msec 2024-11-12T18:14:50,138 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "9911683f163c" port: 45243 start_code: 1731435282573 } transition { transition_code: OPENED region_info { region_id: 1731435284929 table_name { namespace: "default" qualifier: "test" } start_key: "ggg" end_key: "hhh" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 77 initiating_master_active_time: 1731435282704 }; retry (#0) after 1005ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-12T18:14:50,139 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "9911683f163c" port: 34847 start_code: 1731435282512 } transition { transition_code: OPENED region_info { region_id: 1731435284929 table_name { namespace: "default" qualifier: "test" } start_key: "ppp" end_key: "qqq" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 78 initiating_master_active_time: 1731435282704 }; retry (#0) after 1001ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-12T18:14:50,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=65 2024-11-12T18:14:50,146 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9911683f163c,34847,1731435282512, table=hbase:meta, region=1588230740 2024-11-12T18:14:50,147 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9911683f163c,34847,1731435282512, state=OPEN 2024-11-12T18:14:50,149 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,149 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,149 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,149 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,150 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,150 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,153 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN to state=OFFLINE, location=null, table=test, region=26fc65f8daf71d5831131b1bc6746a63 to restore RIT 2024-11-12T18:14:50,153 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN to state=OFFLINE, location=null, table=test, region=56e168ba1a3103978bee5b6c43feba36 to restore RIT 2024-11-12T18:14:50,153 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN to state=OFFLINE, location=null, table=test, region=1b81a9be48fbe50dde523592a87a4611 to restore RIT 2024-11-12T18:14:50,153 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN to state=OFFLINE, location=null, table=test, region=4b703e982cd6bbf72ec2f771b120c649 to restore RIT 2024-11-12T18:14:50,154 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN to state=OFFLINE, location=null, table=test, region=d59e820eb0eef2601f5990d32a8adbc9 to restore RIT 2024-11-12T18:14:50,154 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN to state=OFFLINE, location=null, table=test, region=9fbe24cda440bbf727cd581ab33ad7a9 to restore RIT 2024-11-12T18:14:50,154 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN to state=OFFLINE, location=null, table=test, region=2248a0d1296d1dbe4f8c9805798db03c to restore RIT 2024-11-12T18:14:50,154 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN to state=OFFLINE, location=null, table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a to restore RIT 2024-11-12T18:14:50,164 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731435320164 2024-11-12T18:14:50,165 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-12T18:14:50,165 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,166 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T18:14:50,172 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T18:14:50,172 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=81, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b81a9be48fbe50dde523592a87a4611, server=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,173 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,173 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=81, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b81a9be48fbe50dde523592a87a4611, server=9911683f163c,45243,1731435282573 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,175 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-11-12T18:14:50,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dcfe82d27e00f564e4ad71ea2e78f5c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2248a0d1296d1dbe4f8c9805798db03c, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,207 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a62a194de5221e9f84b2cfd2e83a46a, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,207 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e290ad15dec09a0cb11782988af7c4c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=5 2024-11-12T18:14:50,207 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0b5ce997df05cef2b0f3fd25e49fafc, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,207 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4aff96e33c9923b5d2448f72171fd310, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,208 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26fc65f8daf71d5831131b1bc6746a63, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,208 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dff0a9c660bd5084b34753ea40ea3278, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,208 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96e9ca1050c6b52ea6556fdb4b5d5f39, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a6d3052a2b28af2dab58e9247bb22b7, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d59e820eb0eef2601f5990d32a8adbc9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=394b0963ad0828345e14c3be23cb7790, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c739eeba765df343af4ef63444655c9f, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b703e982cd6bbf72ec2f771b120c649, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b076d46dbfff2e1f3c89ad5c3f814d, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fbe24cda440bbf727cd581ab33ad7a9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d7c21378c2e7788ac010db26909da83, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b81a9be48fbe50dde523592a87a4611, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4795f188f4931ea02df11d7474e4daa4, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e38ac98b32f350d84df37b6915c56a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7b0558136890d45b56827e5af943cfa, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56e168ba1a3103978bee5b6c43feba36, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec76766d593d312811f043efb129a5a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0178ff26f4e2d0ab40d50fe6fcd026be, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fde2b7fbe9fa7c8a87ab544f5542d82a, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e4348dabb8addd0673ec5266827d8b4d, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,214 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-12T18:14:50,214 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731435350214 2024-11-12T18:14:50,214 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731435410214 2024-11-12T18:14:50,214 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 49 msec 2024-11-12T18:14:50,215 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T18:14:50,215 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,215 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T18:14:50,215 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T18:14:50,215 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T18:14:50,215 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T18:14:50,215 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T18:14:50,216 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T18:14:50,217 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T18:14:50,222 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T18:14:50,222 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T18:14:50,222 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T18:14:50,222 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,222 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,222 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,222 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,222 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,222 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,223 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,223 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,223 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,223 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,223 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,223 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,223 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,223 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,223 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,223 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T18:14:50,223 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T18:14:50,223 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,223 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,223 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T18:14:50,225 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T18:14:50,225 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6e290ad15dec09a0cb11782988af7c4c, REOPEN/MOVE 2024-11-12T18:14:50,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true 2024-11-12T18:14:50,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T18:14:50,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN 2024-11-12T18:14:50,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN 2024-11-12T18:14:50,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN 2024-11-12T18:14:50,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN 2024-11-12T18:14:50,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN 2024-11-12T18:14:50,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN 2024-11-12T18:14:50,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN 2024-11-12T18:14:50,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN 2024-11-12T18:14:50,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure fde2b7fbe9fa7c8a87ab544f5542d82a, server=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure d59e820eb0eef2601f5990d32a8adbc9, server=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26fc65f8daf71d5831131b1bc6746a63, server=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9fbe24cda440bbf727cd581ab33ad7a9, server=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 56e168ba1a3103978bee5b6c43feba36, server=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=70, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b81a9be48fbe50dde523592a87a4611, server=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,229 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,230 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,230 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,230 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,230 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,230 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,230 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,230 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,230 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN 2024-11-12T18:14:50,230 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,230 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN 2024-11-12T18:14:50,230 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,230 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN 2024-11-12T18:14:50,230 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,230 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN 2024-11-12T18:14:50,230 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,230 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN 2024-11-12T18:14:50,230 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,230 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN 2024-11-12T18:14:50,230 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,230 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN 2024-11-12T18:14:50,230 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,230 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN 2024-11-12T18:14:50,230 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure fde2b7fbe9fa7c8a87ab544f5542d82a, server=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,230 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure d59e820eb0eef2601f5990d32a8adbc9, server=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,230 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26fc65f8daf71d5831131b1bc6746a63, server=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,230 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9fbe24cda440bbf727cd581ab33ad7a9, server=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,230 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 56e168ba1a3103978bee5b6c43feba36, server=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,231 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=70, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,231 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b81a9be48fbe50dde523592a87a4611, server=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,231 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 13 msec 2024-11-12T18:14:50,232 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=65 2024-11-12T18:14:50,232 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9911683f163c,34847,1731435282512, table=hbase:meta, region=1588230740 2024-11-12T18:14:50,233 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9911683f163c,34847,1731435282512, state=OPEN 2024-11-12T18:14:50,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,237 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,237 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,237 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,238 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN to state=OFFLINE, location=null, table=test, region=26fc65f8daf71d5831131b1bc6746a63 to restore RIT 2024-11-12T18:14:50,238 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN to state=OFFLINE, location=null, table=test, region=56e168ba1a3103978bee5b6c43feba36 to restore RIT 2024-11-12T18:14:50,238 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN to state=OFFLINE, location=null, table=test, region=1b81a9be48fbe50dde523592a87a4611 to restore RIT 2024-11-12T18:14:50,239 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN to state=OFFLINE, location=null, table=test, region=4b703e982cd6bbf72ec2f771b120c649 to restore RIT 2024-11-12T18:14:50,239 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN to state=OFFLINE, location=null, table=test, region=d59e820eb0eef2601f5990d32a8adbc9 to restore RIT 2024-11-12T18:14:50,239 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN to state=OFFLINE, location=null, table=test, region=9fbe24cda440bbf727cd581ab33ad7a9 to restore RIT 2024-11-12T18:14:50,239 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN to state=OFFLINE, location=null, table=test, region=2248a0d1296d1dbe4f8c9805798db03c to restore RIT 2024-11-12T18:14:50,239 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN to state=OFFLINE, location=null, table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a to restore RIT 2024-11-12T18:14:50,240 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731435320240 2024-11-12T18:14:50,240 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-12T18:14:50,240 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,241 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T18:14:50,243 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T18:14:50,243 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=80, ppid=70, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,243 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,243 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=80, ppid=70, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,251 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dcfe82d27e00f564e4ad71ea2e78f5c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2248a0d1296d1dbe4f8c9805798db03c, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a62a194de5221e9f84b2cfd2e83a46a, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e290ad15dec09a0cb11782988af7c4c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=5 2024-11-12T18:14:50,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0b5ce997df05cef2b0f3fd25e49fafc, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4aff96e33c9923b5d2448f72171fd310, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26fc65f8daf71d5831131b1bc6746a63, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dff0a9c660bd5084b34753ea40ea3278, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96e9ca1050c6b52ea6556fdb4b5d5f39, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a6d3052a2b28af2dab58e9247bb22b7, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d59e820eb0eef2601f5990d32a8adbc9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=394b0963ad0828345e14c3be23cb7790, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c739eeba765df343af4ef63444655c9f, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b703e982cd6bbf72ec2f771b120c649, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,256 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b076d46dbfff2e1f3c89ad5c3f814d, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,256 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fbe24cda440bbf727cd581ab33ad7a9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,256 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d7c21378c2e7788ac010db26909da83, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,256 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b81a9be48fbe50dde523592a87a4611, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,257 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4795f188f4931ea02df11d7474e4daa4, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,257 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e38ac98b32f350d84df37b6915c56a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,257 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7b0558136890d45b56827e5af943cfa, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,257 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56e168ba1a3103978bee5b6c43feba36, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,258 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec76766d593d312811f043efb129a5a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,258 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0178ff26f4e2d0ab40d50fe6fcd026be, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,258 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fde2b7fbe9fa7c8a87ab544f5542d82a, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,259 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e4348dabb8addd0673ec5266827d8b4d, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,259 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-12T18:14:50,259 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731435350259 2024-11-12T18:14:50,259 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731435410259 2024-11-12T18:14:50,259 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 19 msec 2024-11-12T18:14:50,259 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T18:14:50,259 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,259 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T18:14:50,259 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T18:14:50,259 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T18:14:50,259 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T18:14:50,259 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T18:14:50,260 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T18:14:50,261 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T18:14:50,266 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T18:14:50,266 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T18:14:50,266 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T18:14:50,266 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,266 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,266 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,266 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,267 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,267 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,267 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,267 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,267 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,267 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,267 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,267 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,267 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,267 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,267 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T18:14:50,267 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T18:14:50,267 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,267 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,267 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T18:14:50,269 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T18:14:50,269 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6e290ad15dec09a0cb11782988af7c4c, REOPEN/MOVE 2024-11-12T18:14:50,273 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true 2024-11-12T18:14:50,273 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,273 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,273 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T18:14:50,273 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,273 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,273 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,273 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN 2024-11-12T18:14:50,273 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN 2024-11-12T18:14:50,273 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN 2024-11-12T18:14:50,273 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN 2024-11-12T18:14:50,273 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN 2024-11-12T18:14:50,274 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN 2024-11-12T18:14:50,274 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN 2024-11-12T18:14:50,274 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN 2024-11-12T18:14:50,274 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure fde2b7fbe9fa7c8a87ab544f5542d82a, server=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,274 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure d59e820eb0eef2601f5990d32a8adbc9, server=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,274 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26fc65f8daf71d5831131b1bc6746a63, server=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,274 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9fbe24cda440bbf727cd581ab33ad7a9, server=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,274 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 56e168ba1a3103978bee5b6c43feba36, server=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,274 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=70, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,274 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,274 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,274 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,274 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,274 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,274 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,274 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,274 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,274 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN 2024-11-12T18:14:50,274 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,274 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN 2024-11-12T18:14:50,274 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,274 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN 2024-11-12T18:14:50,274 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,274 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN 2024-11-12T18:14:50,274 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,274 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN 2024-11-12T18:14:50,274 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,274 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN 2024-11-12T18:14:50,274 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,275 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN 2024-11-12T18:14:50,275 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,275 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN 2024-11-12T18:14:50,275 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure fde2b7fbe9fa7c8a87ab544f5542d82a, server=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,275 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure d59e820eb0eef2601f5990d32a8adbc9, server=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,275 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26fc65f8daf71d5831131b1bc6746a63, server=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,275 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9fbe24cda440bbf727cd581ab33ad7a9, server=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,275 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 56e168ba1a3103978bee5b6c43feba36, server=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,275 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=70, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,275 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 14 msec 2024-11-12T18:14:50,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=65 2024-11-12T18:14:50,276 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9911683f163c,34847,1731435282512, table=hbase:meta, region=1588230740 2024-11-12T18:14:50,277 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9911683f163c,34847,1731435282512, state=OPEN 2024-11-12T18:14:50,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,279 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,279 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,279 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,281 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN to state=OFFLINE, location=null, table=test, region=26fc65f8daf71d5831131b1bc6746a63 to restore RIT 2024-11-12T18:14:50,281 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN to state=OFFLINE, location=null, table=test, region=56e168ba1a3103978bee5b6c43feba36 to restore RIT 2024-11-12T18:14:50,281 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN to state=OFFLINE, location=null, table=test, region=1b81a9be48fbe50dde523592a87a4611 to restore RIT 2024-11-12T18:14:50,281 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN to state=OFFLINE, location=null, table=test, region=4b703e982cd6bbf72ec2f771b120c649 to restore RIT 2024-11-12T18:14:50,281 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN to state=OFFLINE, location=null, table=test, region=d59e820eb0eef2601f5990d32a8adbc9 to restore RIT 2024-11-12T18:14:50,281 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN to state=OFFLINE, location=null, table=test, region=9fbe24cda440bbf727cd581ab33ad7a9 to restore RIT 2024-11-12T18:14:50,281 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN to state=OFFLINE, location=null, table=test, region=2248a0d1296d1dbe4f8c9805798db03c to restore RIT 2024-11-12T18:14:50,281 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN to state=OFFLINE, location=null, table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a to restore RIT 2024-11-12T18:14:50,282 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731435320282 2024-11-12T18:14:50,283 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-12T18:14:50,283 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,284 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T18:14:50,285 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T18:14:50,285 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 56e168ba1a3103978bee5b6c43feba36, server=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,285 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,286 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 56e168ba1a3103978bee5b6c43feba36, server=9911683f163c,34847,1731435282512 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,293 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dcfe82d27e00f564e4ad71ea2e78f5c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,294 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2248a0d1296d1dbe4f8c9805798db03c, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,294 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a62a194de5221e9f84b2cfd2e83a46a, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,294 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e290ad15dec09a0cb11782988af7c4c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=5 2024-11-12T18:14:50,294 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0b5ce997df05cef2b0f3fd25e49fafc, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,295 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4aff96e33c9923b5d2448f72171fd310, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,295 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26fc65f8daf71d5831131b1bc6746a63, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,295 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dff0a9c660bd5084b34753ea40ea3278, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,295 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96e9ca1050c6b52ea6556fdb4b5d5f39, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a6d3052a2b28af2dab58e9247bb22b7, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d59e820eb0eef2601f5990d32a8adbc9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=394b0963ad0828345e14c3be23cb7790, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,297 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c739eeba765df343af4ef63444655c9f, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,297 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b703e982cd6bbf72ec2f771b120c649, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,297 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b076d46dbfff2e1f3c89ad5c3f814d, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,298 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fbe24cda440bbf727cd581ab33ad7a9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,298 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d7c21378c2e7788ac010db26909da83, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,299 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b81a9be48fbe50dde523592a87a4611, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,299 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4795f188f4931ea02df11d7474e4daa4, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,299 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e38ac98b32f350d84df37b6915c56a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,300 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7b0558136890d45b56827e5af943cfa, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,300 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56e168ba1a3103978bee5b6c43feba36, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,300 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec76766d593d312811f043efb129a5a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,301 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0178ff26f4e2d0ab40d50fe6fcd026be, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,301 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fde2b7fbe9fa7c8a87ab544f5542d82a, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,301 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e4348dabb8addd0673ec5266827d8b4d, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,301 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-12T18:14:50,301 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731435350301 2024-11-12T18:14:50,301 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731435410301 2024-11-12T18:14:50,301 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 18 msec 2024-11-12T18:14:50,301 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T18:14:50,301 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,301 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T18:14:50,302 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T18:14:50,302 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36779 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,303 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T18:14:50,303 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T18:14:50,303 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T18:14:50,304 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T18:14:50,305 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-12T18:14:50,309 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T18:14:50,309 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T18:14:50,309 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T18:14:50,309 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,309 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,309 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,309 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,309 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,309 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,309 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,309 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,309 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,309 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,309 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,309 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,309 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,309 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T18:14:50,309 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T18:14:50,309 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,309 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,310 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T18:14:50,311 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T18:14:50,311 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6e290ad15dec09a0cb11782988af7c4c, REOPEN/MOVE 2024-11-12T18:14:50,314 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true 2024-11-12T18:14:50,314 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,314 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,314 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T18:14:50,314 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,314 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,314 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,314 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN 2024-11-12T18:14:50,314 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN 2024-11-12T18:14:50,314 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN 2024-11-12T18:14:50,314 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN 2024-11-12T18:14:50,314 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN 2024-11-12T18:14:50,314 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN 2024-11-12T18:14:50,314 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN 2024-11-12T18:14:50,314 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN 2024-11-12T18:14:50,314 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure fde2b7fbe9fa7c8a87ab544f5542d82a, server=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,314 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure d59e820eb0eef2601f5990d32a8adbc9, server=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,314 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26fc65f8daf71d5831131b1bc6746a63, server=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,314 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9fbe24cda440bbf727cd581ab33ad7a9, server=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,314 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 56e168ba1a3103978bee5b6c43feba36, server=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,314 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,315 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,315 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,315 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,315 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,315 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,315 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,315 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,315 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN 2024-11-12T18:14:50,315 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,315 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN 2024-11-12T18:14:50,315 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,315 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN 2024-11-12T18:14:50,315 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,315 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN 2024-11-12T18:14:50,315 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,315 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN 2024-11-12T18:14:50,315 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,315 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN 2024-11-12T18:14:50,315 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,315 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN 2024-11-12T18:14:50,315 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,315 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN 2024-11-12T18:14:50,315 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure fde2b7fbe9fa7c8a87ab544f5542d82a, server=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,315 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure d59e820eb0eef2601f5990d32a8adbc9, server=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,315 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26fc65f8daf71d5831131b1bc6746a63, server=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,315 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9fbe24cda440bbf727cd581ab33ad7a9, server=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,315 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 56e168ba1a3103978bee5b6c43feba36, server=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,316 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-12T18:14:50,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=65 2024-11-12T18:14:50,317 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9911683f163c,34847,1731435282512, table=hbase:meta, region=1588230740 2024-11-12T18:14:50,317 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9911683f163c,34847,1731435282512, state=OPEN 2024-11-12T18:14:50,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,319 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,319 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,319 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,320 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN to state=OFFLINE, location=null, table=test, region=26fc65f8daf71d5831131b1bc6746a63 to restore RIT 2024-11-12T18:14:50,320 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN to state=OFFLINE, location=null, table=test, region=56e168ba1a3103978bee5b6c43feba36 to restore RIT 2024-11-12T18:14:50,320 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN to state=OFFLINE, location=null, table=test, region=1b81a9be48fbe50dde523592a87a4611 to restore RIT 2024-11-12T18:14:50,320 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN to state=OFFLINE, location=null, table=test, region=4b703e982cd6bbf72ec2f771b120c649 to restore RIT 2024-11-12T18:14:50,320 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN to state=OFFLINE, location=null, table=test, region=d59e820eb0eef2601f5990d32a8adbc9 to restore RIT 2024-11-12T18:14:50,320 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN to state=OFFLINE, location=null, table=test, region=9fbe24cda440bbf727cd581ab33ad7a9 to restore RIT 2024-11-12T18:14:50,320 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN to state=OFFLINE, location=null, table=test, region=2248a0d1296d1dbe4f8c9805798db03c to restore RIT 2024-11-12T18:14:50,320 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN to state=OFFLINE, location=null, table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a to restore RIT 2024-11-12T18:14:50,321 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731435320321 2024-11-12T18:14:50,321 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-12T18:14:50,321 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,322 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T18:14:50,323 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T18:14:50,323 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9fbe24cda440bbf727cd581ab33ad7a9, server=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,323 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,323 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9fbe24cda440bbf727cd581ab33ad7a9, server=9911683f163c,34847,1731435282512 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,327 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dcfe82d27e00f564e4ad71ea2e78f5c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,327 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2248a0d1296d1dbe4f8c9805798db03c, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,328 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a62a194de5221e9f84b2cfd2e83a46a, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,328 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e290ad15dec09a0cb11782988af7c4c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=5 2024-11-12T18:14:50,328 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0b5ce997df05cef2b0f3fd25e49fafc, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,328 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4aff96e33c9923b5d2448f72171fd310, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,328 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26fc65f8daf71d5831131b1bc6746a63, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,329 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dff0a9c660bd5084b34753ea40ea3278, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,329 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96e9ca1050c6b52ea6556fdb4b5d5f39, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,329 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a6d3052a2b28af2dab58e9247bb22b7, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,329 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d59e820eb0eef2601f5990d32a8adbc9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,330 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=394b0963ad0828345e14c3be23cb7790, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,330 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c739eeba765df343af4ef63444655c9f, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,330 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b703e982cd6bbf72ec2f771b120c649, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,330 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b076d46dbfff2e1f3c89ad5c3f814d, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,330 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fbe24cda440bbf727cd581ab33ad7a9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,331 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d7c21378c2e7788ac010db26909da83, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,331 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b81a9be48fbe50dde523592a87a4611, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,331 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4795f188f4931ea02df11d7474e4daa4, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,331 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e38ac98b32f350d84df37b6915c56a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,331 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7b0558136890d45b56827e5af943cfa, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,332 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56e168ba1a3103978bee5b6c43feba36, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,332 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec76766d593d312811f043efb129a5a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,332 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0178ff26f4e2d0ab40d50fe6fcd026be, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,332 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fde2b7fbe9fa7c8a87ab544f5542d82a, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,333 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e4348dabb8addd0673ec5266827d8b4d, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,333 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-12T18:14:50,333 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731435350333 2024-11-12T18:14:50,333 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731435410333 2024-11-12T18:14:50,333 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 11 msec 2024-11-12T18:14:50,333 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T18:14:50,333 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,333 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T18:14:50,333 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T18:14:50,333 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T18:14:50,333 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T18:14:50,334 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T18:14:50,334 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T18:14:50,335 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T18:14:50,337 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T18:14:50,337 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T18:14:50,337 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T18:14:50,337 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,337 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,337 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,337 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,337 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,337 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,337 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,337 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,337 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,337 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,337 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,337 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,337 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T18:14:50,337 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T18:14:50,337 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,337 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,337 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T18:14:50,338 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T18:14:50,338 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6e290ad15dec09a0cb11782988af7c4c, REOPEN/MOVE 2024-11-12T18:14:50,340 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true 2024-11-12T18:14:50,340 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,340 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,340 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T18:14:50,340 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,340 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,340 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,340 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN 2024-11-12T18:14:50,340 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN 2024-11-12T18:14:50,340 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN 2024-11-12T18:14:50,340 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN 2024-11-12T18:14:50,340 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN 2024-11-12T18:14:50,340 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN 2024-11-12T18:14:50,340 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN 2024-11-12T18:14:50,340 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN 2024-11-12T18:14:50,340 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure fde2b7fbe9fa7c8a87ab544f5542d82a, server=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,340 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure d59e820eb0eef2601f5990d32a8adbc9, server=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,340 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26fc65f8daf71d5831131b1bc6746a63, server=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,340 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9fbe24cda440bbf727cd581ab33ad7a9, server=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,340 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,340 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,340 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,340 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,340 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,341 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,341 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,341 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,341 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN 2024-11-12T18:14:50,341 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,341 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN 2024-11-12T18:14:50,341 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,341 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN 2024-11-12T18:14:50,341 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,341 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN 2024-11-12T18:14:50,341 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,341 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN 2024-11-12T18:14:50,341 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,341 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN 2024-11-12T18:14:50,341 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,341 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN 2024-11-12T18:14:50,341 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,341 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN 2024-11-12T18:14:50,341 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure fde2b7fbe9fa7c8a87ab544f5542d82a, server=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,341 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure d59e820eb0eef2601f5990d32a8adbc9, server=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,341 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26fc65f8daf71d5831131b1bc6746a63, server=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,341 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9fbe24cda440bbf727cd581ab33ad7a9, server=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,341 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-12T18:14:50,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=65 2024-11-12T18:14:50,342 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9911683f163c,34847,1731435282512, table=hbase:meta, region=1588230740 2024-11-12T18:14:50,342 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9911683f163c,34847,1731435282512, state=OPEN 2024-11-12T18:14:50,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,344 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,344 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,344 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,345 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN to state=OFFLINE, location=null, table=test, region=26fc65f8daf71d5831131b1bc6746a63 to restore RIT 2024-11-12T18:14:50,345 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN to state=OFFLINE, location=null, table=test, region=56e168ba1a3103978bee5b6c43feba36 to restore RIT 2024-11-12T18:14:50,345 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN to state=OFFLINE, location=null, table=test, region=1b81a9be48fbe50dde523592a87a4611 to restore RIT 2024-11-12T18:14:50,345 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN to state=OFFLINE, location=null, table=test, region=4b703e982cd6bbf72ec2f771b120c649 to restore RIT 2024-11-12T18:14:50,345 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN to state=OFFLINE, location=null, table=test, region=d59e820eb0eef2601f5990d32a8adbc9 to restore RIT 2024-11-12T18:14:50,346 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN to state=OFFLINE, location=null, table=test, region=9fbe24cda440bbf727cd581ab33ad7a9 to restore RIT 2024-11-12T18:14:50,346 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN to state=OFFLINE, location=null, table=test, region=2248a0d1296d1dbe4f8c9805798db03c to restore RIT 2024-11-12T18:14:50,346 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN to state=OFFLINE, location=null, table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a to restore RIT 2024-11-12T18:14:50,348 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731435320348 2024-11-12T18:14:50,348 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-12T18:14:50,349 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,350 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T18:14:50,352 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T18:14:50,352 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26fc65f8daf71d5831131b1bc6746a63, server=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,352 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,352 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26fc65f8daf71d5831131b1bc6746a63, server=9911683f163c,45243,1731435282573 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,354 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dcfe82d27e00f564e4ad71ea2e78f5c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,354 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2248a0d1296d1dbe4f8c9805798db03c, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,355 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a62a194de5221e9f84b2cfd2e83a46a, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,355 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e290ad15dec09a0cb11782988af7c4c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=5 2024-11-12T18:14:50,355 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0b5ce997df05cef2b0f3fd25e49fafc, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,355 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4aff96e33c9923b5d2448f72171fd310, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,355 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26fc65f8daf71d5831131b1bc6746a63, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,355 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dff0a9c660bd5084b34753ea40ea3278, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,355 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96e9ca1050c6b52ea6556fdb4b5d5f39, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,356 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a6d3052a2b28af2dab58e9247bb22b7, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,356 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d59e820eb0eef2601f5990d32a8adbc9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,356 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=394b0963ad0828345e14c3be23cb7790, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,356 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c739eeba765df343af4ef63444655c9f, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,356 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b703e982cd6bbf72ec2f771b120c649, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,356 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b076d46dbfff2e1f3c89ad5c3f814d, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,357 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fbe24cda440bbf727cd581ab33ad7a9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,357 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d7c21378c2e7788ac010db26909da83, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,357 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b81a9be48fbe50dde523592a87a4611, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,357 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4795f188f4931ea02df11d7474e4daa4, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,357 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e38ac98b32f350d84df37b6915c56a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,357 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7b0558136890d45b56827e5af943cfa, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,358 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56e168ba1a3103978bee5b6c43feba36, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,358 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec76766d593d312811f043efb129a5a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,358 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0178ff26f4e2d0ab40d50fe6fcd026be, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,358 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fde2b7fbe9fa7c8a87ab544f5542d82a, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,358 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e4348dabb8addd0673ec5266827d8b4d, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,358 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-12T18:14:50,358 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731435350358 2024-11-12T18:14:50,358 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731435410358 2024-11-12T18:14:50,358 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-12T18:14:50,358 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T18:14:50,358 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,358 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T18:14:50,358 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T18:14:50,359 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T18:14:50,359 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T18:14:50,359 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T18:14:50,359 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T18:14:50,360 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T18:14:50,362 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T18:14:50,362 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T18:14:50,362 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T18:14:50,362 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,362 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,362 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,362 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,362 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,362 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,362 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,362 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,362 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,362 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,362 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,362 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T18:14:50,362 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T18:14:50,362 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,362 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,362 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T18:14:50,363 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T18:14:50,363 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6e290ad15dec09a0cb11782988af7c4c, REOPEN/MOVE 2024-11-12T18:14:50,364 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true 2024-11-12T18:14:50,364 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,364 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,365 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T18:14:50,365 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,365 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,365 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,365 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN 2024-11-12T18:14:50,365 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN 2024-11-12T18:14:50,365 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN 2024-11-12T18:14:50,365 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN 2024-11-12T18:14:50,365 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN 2024-11-12T18:14:50,365 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN 2024-11-12T18:14:50,365 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN 2024-11-12T18:14:50,365 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN 2024-11-12T18:14:50,365 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure fde2b7fbe9fa7c8a87ab544f5542d82a, server=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,365 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure d59e820eb0eef2601f5990d32a8adbc9, server=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,365 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26fc65f8daf71d5831131b1bc6746a63, server=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,365 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,365 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,365 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,365 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,365 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,365 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,365 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,365 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,365 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN 2024-11-12T18:14:50,365 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,365 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN 2024-11-12T18:14:50,365 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,365 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN 2024-11-12T18:14:50,365 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,365 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN 2024-11-12T18:14:50,365 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,365 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN 2024-11-12T18:14:50,365 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,365 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN 2024-11-12T18:14:50,365 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,365 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN 2024-11-12T18:14:50,365 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,366 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN 2024-11-12T18:14:50,366 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure fde2b7fbe9fa7c8a87ab544f5542d82a, server=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,366 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure d59e820eb0eef2601f5990d32a8adbc9, server=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,366 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26fc65f8daf71d5831131b1bc6746a63, server=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,366 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-12T18:14:50,366 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=65 2024-11-12T18:14:50,367 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9911683f163c,34847,1731435282512, table=hbase:meta, region=1588230740 2024-11-12T18:14:50,367 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9911683f163c,34847,1731435282512, state=OPEN 2024-11-12T18:14:50,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,369 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,369 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,369 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,369 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN to state=OFFLINE, location=null, table=test, region=26fc65f8daf71d5831131b1bc6746a63 to restore RIT 2024-11-12T18:14:50,370 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN to state=OFFLINE, location=null, table=test, region=56e168ba1a3103978bee5b6c43feba36 to restore RIT 2024-11-12T18:14:50,370 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN to state=OFFLINE, location=null, table=test, region=1b81a9be48fbe50dde523592a87a4611 to restore RIT 2024-11-12T18:14:50,370 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN to state=OFFLINE, location=null, table=test, region=4b703e982cd6bbf72ec2f771b120c649 to restore RIT 2024-11-12T18:14:50,370 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN to state=OFFLINE, location=null, table=test, region=d59e820eb0eef2601f5990d32a8adbc9 to restore RIT 2024-11-12T18:14:50,370 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN to state=OFFLINE, location=null, table=test, region=9fbe24cda440bbf727cd581ab33ad7a9 to restore RIT 2024-11-12T18:14:50,370 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN to state=OFFLINE, location=null, table=test, region=2248a0d1296d1dbe4f8c9805798db03c to restore RIT 2024-11-12T18:14:50,370 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN to state=OFFLINE, location=null, table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a to restore RIT 2024-11-12T18:14:50,370 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731435320370 2024-11-12T18:14:50,370 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-12T18:14:50,370 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,372 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T18:14:50,373 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T18:14:50,373 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure d59e820eb0eef2601f5990d32a8adbc9, server=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,373 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,373 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure d59e820eb0eef2601f5990d32a8adbc9, server=9911683f163c,34847,1731435282512 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,377 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dcfe82d27e00f564e4ad71ea2e78f5c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,378 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2248a0d1296d1dbe4f8c9805798db03c, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,378 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a62a194de5221e9f84b2cfd2e83a46a, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,378 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e290ad15dec09a0cb11782988af7c4c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=5 2024-11-12T18:14:50,378 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0b5ce997df05cef2b0f3fd25e49fafc, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,378 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4aff96e33c9923b5d2448f72171fd310, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,378 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26fc65f8daf71d5831131b1bc6746a63, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,379 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dff0a9c660bd5084b34753ea40ea3278, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,379 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96e9ca1050c6b52ea6556fdb4b5d5f39, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,379 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a6d3052a2b28af2dab58e9247bb22b7, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,379 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d59e820eb0eef2601f5990d32a8adbc9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,379 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=394b0963ad0828345e14c3be23cb7790, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,379 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c739eeba765df343af4ef63444655c9f, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,379 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b703e982cd6bbf72ec2f771b120c649, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,380 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b076d46dbfff2e1f3c89ad5c3f814d, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,380 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fbe24cda440bbf727cd581ab33ad7a9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,380 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d7c21378c2e7788ac010db26909da83, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,380 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b81a9be48fbe50dde523592a87a4611, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,380 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4795f188f4931ea02df11d7474e4daa4, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,380 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e38ac98b32f350d84df37b6915c56a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,380 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7b0558136890d45b56827e5af943cfa, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56e168ba1a3103978bee5b6c43feba36, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec76766d593d312811f043efb129a5a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0178ff26f4e2d0ab40d50fe6fcd026be, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fde2b7fbe9fa7c8a87ab544f5542d82a, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e4348dabb8addd0673ec5266827d8b4d, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,381 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-12T18:14:50,381 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731435350381 2024-11-12T18:14:50,381 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731435410381 2024-11-12T18:14:50,381 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 11 msec 2024-11-12T18:14:50,381 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T18:14:50,381 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,382 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T18:14:50,382 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T18:14:50,382 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T18:14:50,382 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T18:14:50,382 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T18:14:50,382 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T18:14:50,383 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T18:14:50,385 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T18:14:50,385 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T18:14:50,385 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T18:14:50,385 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,385 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,385 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,385 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,385 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,385 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,385 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,385 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,385 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,385 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,385 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T18:14:50,385 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T18:14:50,385 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,385 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,385 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T18:14:50,386 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T18:14:50,386 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6e290ad15dec09a0cb11782988af7c4c, REOPEN/MOVE 2024-11-12T18:14:50,387 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true 2024-11-12T18:14:50,388 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,388 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,388 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T18:14:50,388 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,388 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,388 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,388 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN 2024-11-12T18:14:50,388 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN 2024-11-12T18:14:50,388 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN 2024-11-12T18:14:50,388 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN 2024-11-12T18:14:50,388 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN 2024-11-12T18:14:50,388 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN 2024-11-12T18:14:50,388 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN 2024-11-12T18:14:50,388 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN 2024-11-12T18:14:50,388 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure fde2b7fbe9fa7c8a87ab544f5542d82a, server=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,388 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure d59e820eb0eef2601f5990d32a8adbc9, server=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,388 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,388 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,388 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,388 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,388 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,388 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,388 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,388 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,388 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN 2024-11-12T18:14:50,388 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,388 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN 2024-11-12T18:14:50,388 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,388 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN 2024-11-12T18:14:50,388 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,388 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN 2024-11-12T18:14:50,388 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,388 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN 2024-11-12T18:14:50,388 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,388 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN 2024-11-12T18:14:50,388 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,388 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN 2024-11-12T18:14:50,388 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,388 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN 2024-11-12T18:14:50,388 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure fde2b7fbe9fa7c8a87ab544f5542d82a, server=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,389 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure d59e820eb0eef2601f5990d32a8adbc9, server=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,389 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-12T18:14:50,390 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=65 2024-11-12T18:14:50,390 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9911683f163c,34847,1731435282512, table=hbase:meta, region=1588230740 2024-11-12T18:14:50,390 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9911683f163c,34847,1731435282512, state=OPEN 2024-11-12T18:14:50,393 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,393 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,393 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,393 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,397 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,398 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN to state=OFFLINE, location=null, table=test, region=26fc65f8daf71d5831131b1bc6746a63 to restore RIT 2024-11-12T18:14:50,398 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN to state=OFFLINE, location=null, table=test, region=56e168ba1a3103978bee5b6c43feba36 to restore RIT 2024-11-12T18:14:50,398 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN to state=OFFLINE, location=null, table=test, region=1b81a9be48fbe50dde523592a87a4611 to restore RIT 2024-11-12T18:14:50,398 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN to state=OFFLINE, location=null, table=test, region=4b703e982cd6bbf72ec2f771b120c649 to restore RIT 2024-11-12T18:14:50,398 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN to state=OFFLINE, location=null, table=test, region=d59e820eb0eef2601f5990d32a8adbc9 to restore RIT 2024-11-12T18:14:50,398 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN to state=OFFLINE, location=null, table=test, region=9fbe24cda440bbf727cd581ab33ad7a9 to restore RIT 2024-11-12T18:14:50,398 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN to state=OFFLINE, location=null, table=test, region=2248a0d1296d1dbe4f8c9805798db03c to restore RIT 2024-11-12T18:14:50,398 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN to state=OFFLINE, location=null, table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a to restore RIT 2024-11-12T18:14:50,400 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731435320400 2024-11-12T18:14:50,401 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-12T18:14:50,402 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,403 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T18:14:50,406 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T18:14:50,406 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure fde2b7fbe9fa7c8a87ab544f5542d82a, server=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,406 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,406 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure fde2b7fbe9fa7c8a87ab544f5542d82a, server=9911683f163c,45243,1731435282573 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,412 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dcfe82d27e00f564e4ad71ea2e78f5c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,413 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2248a0d1296d1dbe4f8c9805798db03c, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,413 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a62a194de5221e9f84b2cfd2e83a46a, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,413 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e290ad15dec09a0cb11782988af7c4c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=5 2024-11-12T18:14:50,413 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0b5ce997df05cef2b0f3fd25e49fafc, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,413 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4aff96e33c9923b5d2448f72171fd310, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26fc65f8daf71d5831131b1bc6746a63, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dff0a9c660bd5084b34753ea40ea3278, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96e9ca1050c6b52ea6556fdb4b5d5f39, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a6d3052a2b28af2dab58e9247bb22b7, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d59e820eb0eef2601f5990d32a8adbc9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=394b0963ad0828345e14c3be23cb7790, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c739eeba765df343af4ef63444655c9f, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b703e982cd6bbf72ec2f771b120c649, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b076d46dbfff2e1f3c89ad5c3f814d, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fbe24cda440bbf727cd581ab33ad7a9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d7c21378c2e7788ac010db26909da83, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,416 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b81a9be48fbe50dde523592a87a4611, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,416 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4795f188f4931ea02df11d7474e4daa4, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,416 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e38ac98b32f350d84df37b6915c56a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,416 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7b0558136890d45b56827e5af943cfa, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,416 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56e168ba1a3103978bee5b6c43feba36, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,417 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec76766d593d312811f043efb129a5a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,417 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0178ff26f4e2d0ab40d50fe6fcd026be, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,417 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fde2b7fbe9fa7c8a87ab544f5542d82a, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,417 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e4348dabb8addd0673ec5266827d8b4d, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,417 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-12T18:14:50,417 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731435350417 2024-11-12T18:14:50,417 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731435410417 2024-11-12T18:14:50,417 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 16 msec 2024-11-12T18:14:50,418 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T18:14:50,418 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,418 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T18:14:50,418 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T18:14:50,418 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T18:14:50,418 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T18:14:50,418 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T18:14:50,419 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T18:14:50,420 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-12T18:14:50,423 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T18:14:50,423 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T18:14:50,423 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T18:14:50,423 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,423 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,423 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,423 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,423 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,423 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,423 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,423 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,423 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,423 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T18:14:50,423 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T18:14:50,423 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,423 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,424 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T18:14:50,425 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T18:14:50,425 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6e290ad15dec09a0cb11782988af7c4c, REOPEN/MOVE 2024-11-12T18:14:50,427 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true 2024-11-12T18:14:50,427 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,428 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,428 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T18:14:50,428 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,428 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,428 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,428 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN 2024-11-12T18:14:50,428 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN 2024-11-12T18:14:50,428 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN 2024-11-12T18:14:50,428 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN 2024-11-12T18:14:50,428 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN 2024-11-12T18:14:50,428 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN 2024-11-12T18:14:50,428 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN 2024-11-12T18:14:50,428 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN 2024-11-12T18:14:50,428 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure fde2b7fbe9fa7c8a87ab544f5542d82a, server=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,428 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,428 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,428 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,428 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,428 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,428 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,428 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,428 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,428 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN 2024-11-12T18:14:50,428 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,428 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN 2024-11-12T18:14:50,429 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,429 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN 2024-11-12T18:14:50,429 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,429 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN 2024-11-12T18:14:50,429 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,429 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN 2024-11-12T18:14:50,429 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,429 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN 2024-11-12T18:14:50,429 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,429 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN 2024-11-12T18:14:50,429 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,429 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN 2024-11-12T18:14:50,429 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure fde2b7fbe9fa7c8a87ab544f5542d82a, server=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,429 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-12T18:14:50,430 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=65 2024-11-12T18:14:50,430 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9911683f163c,34847,1731435282512, table=hbase:meta, region=1588230740 2024-11-12T18:14:50,431 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9911683f163c,34847,1731435282512, state=OPEN 2024-11-12T18:14:50,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,433 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,433 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,433 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,434 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN to state=OFFLINE, location=null, table=test, region=26fc65f8daf71d5831131b1bc6746a63 to restore RIT 2024-11-12T18:14:50,434 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN to state=OFFLINE, location=null, table=test, region=56e168ba1a3103978bee5b6c43feba36 to restore RIT 2024-11-12T18:14:50,434 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN to state=OFFLINE, location=null, table=test, region=1b81a9be48fbe50dde523592a87a4611 to restore RIT 2024-11-12T18:14:50,435 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN to state=OFFLINE, location=null, table=test, region=4b703e982cd6bbf72ec2f771b120c649 to restore RIT 2024-11-12T18:14:50,435 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN to state=OFFLINE, location=null, table=test, region=d59e820eb0eef2601f5990d32a8adbc9 to restore RIT 2024-11-12T18:14:50,435 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN to state=OFFLINE, location=null, table=test, region=9fbe24cda440bbf727cd581ab33ad7a9 to restore RIT 2024-11-12T18:14:50,435 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN to state=OFFLINE, location=null, table=test, region=2248a0d1296d1dbe4f8c9805798db03c to restore RIT 2024-11-12T18:14:50,435 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN to state=OFFLINE, location=null, table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a to restore RIT 2024-11-12T18:14:50,435 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731435320435 2024-11-12T18:14:50,435 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-12T18:14:50,435 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,437 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T18:14:50,441 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T18:14:50,441 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN 2024-11-12T18:14:50,441 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,441 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,442 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dcfe82d27e00f564e4ad71ea2e78f5c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,442 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2248a0d1296d1dbe4f8c9805798db03c, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,443 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a62a194de5221e9f84b2cfd2e83a46a, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,443 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e290ad15dec09a0cb11782988af7c4c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=5 2024-11-12T18:14:50,443 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0b5ce997df05cef2b0f3fd25e49fafc, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,443 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4aff96e33c9923b5d2448f72171fd310, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,443 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26fc65f8daf71d5831131b1bc6746a63, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,444 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dff0a9c660bd5084b34753ea40ea3278, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,444 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96e9ca1050c6b52ea6556fdb4b5d5f39, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,444 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a6d3052a2b28af2dab58e9247bb22b7, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,444 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d59e820eb0eef2601f5990d32a8adbc9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,444 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=394b0963ad0828345e14c3be23cb7790, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,445 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c739eeba765df343af4ef63444655c9f, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,445 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b703e982cd6bbf72ec2f771b120c649, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,445 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b076d46dbfff2e1f3c89ad5c3f814d, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,445 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fbe24cda440bbf727cd581ab33ad7a9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,445 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d7c21378c2e7788ac010db26909da83, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,446 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b81a9be48fbe50dde523592a87a4611, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,446 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4795f188f4931ea02df11d7474e4daa4, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,446 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e38ac98b32f350d84df37b6915c56a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,446 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7b0558136890d45b56827e5af943cfa, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,446 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56e168ba1a3103978bee5b6c43feba36, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,446 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec76766d593d312811f043efb129a5a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,447 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0178ff26f4e2d0ab40d50fe6fcd026be, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,447 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fde2b7fbe9fa7c8a87ab544f5542d82a, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,447 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e4348dabb8addd0673ec5266827d8b4d, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,447 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-12T18:14:50,447 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731435350447 2024-11-12T18:14:50,447 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731435410447 2024-11-12T18:14:50,447 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 11 msec 2024-11-12T18:14:50,447 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T18:14:50,448 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,448 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T18:14:50,448 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T18:14:50,448 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T18:14:50,448 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T18:14:50,448 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T18:14:50,448 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T18:14:50,449 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T18:14:50,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T18:14:50,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T18:14:50,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T18:14:50,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T18:14:50,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T18:14:50,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,453 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T18:14:50,453 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T18:14:50,454 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6e290ad15dec09a0cb11782988af7c4c, REOPEN/MOVE 2024-11-12T18:14:50,455 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true 2024-11-12T18:14:50,455 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,455 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,455 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T18:14:50,455 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,455 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,455 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN 2024-11-12T18:14:50,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN 2024-11-12T18:14:50,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN 2024-11-12T18:14:50,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN 2024-11-12T18:14:50,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN 2024-11-12T18:14:50,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN 2024-11-12T18:14:50,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN 2024-11-12T18:14:50,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN 2024-11-12T18:14:50,456 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,456 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,456 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,456 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,456 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,456 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,456 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,456 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,456 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN 2024-11-12T18:14:50,456 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,456 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN 2024-11-12T18:14:50,457 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,457 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN 2024-11-12T18:14:50,457 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,457 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN 2024-11-12T18:14:50,457 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,457 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN 2024-11-12T18:14:50,457 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,457 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN 2024-11-12T18:14:50,457 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,457 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN 2024-11-12T18:14:50,457 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,457 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN 2024-11-12T18:14:50,457 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-12T18:14:50,458 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=65 2024-11-12T18:14:50,458 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9911683f163c,34847,1731435282512, table=hbase:meta, region=1588230740 2024-11-12T18:14:50,459 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9911683f163c,34847,1731435282512, state=OPEN 2024-11-12T18:14:50,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,461 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,461 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,461 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,462 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN to state=OFFLINE, location=null, table=test, region=26fc65f8daf71d5831131b1bc6746a63 to restore RIT 2024-11-12T18:14:50,462 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN to state=OFFLINE, location=null, table=test, region=56e168ba1a3103978bee5b6c43feba36 to restore RIT 2024-11-12T18:14:50,462 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN to state=OFFLINE, location=null, table=test, region=1b81a9be48fbe50dde523592a87a4611 to restore RIT 2024-11-12T18:14:50,462 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN to state=OFFLINE, location=null, table=test, region=4b703e982cd6bbf72ec2f771b120c649 to restore RIT 2024-11-12T18:14:50,462 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN to state=OFFLINE, location=null, table=test, region=d59e820eb0eef2601f5990d32a8adbc9 to restore RIT 2024-11-12T18:14:50,462 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN to state=OFFLINE, location=null, table=test, region=9fbe24cda440bbf727cd581ab33ad7a9 to restore RIT 2024-11-12T18:14:50,463 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN to state=OFFLINE, location=null, table=test, region=2248a0d1296d1dbe4f8c9805798db03c to restore RIT 2024-11-12T18:14:50,463 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a, ASSIGN to state=OFFLINE, location=null, table=test, region=fde2b7fbe9fa7c8a87ab544f5542d82a to restore RIT 2024-11-12T18:14:50,463 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731435320463 2024-11-12T18:14:50,464 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-12T18:14:50,464 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,465 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T18:14:50,469 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T18:14:50,469 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN 2024-11-12T18:14:50,469 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dcfe82d27e00f564e4ad71ea2e78f5c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2248a0d1296d1dbe4f8c9805798db03c, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a62a194de5221e9f84b2cfd2e83a46a, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e290ad15dec09a0cb11782988af7c4c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=5 2024-11-12T18:14:50,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0b5ce997df05cef2b0f3fd25e49fafc, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4aff96e33c9923b5d2448f72171fd310, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26fc65f8daf71d5831131b1bc6746a63, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dff0a9c660bd5084b34753ea40ea3278, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,469 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96e9ca1050c6b52ea6556fdb4b5d5f39, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a6d3052a2b28af2dab58e9247bb22b7, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d59e820eb0eef2601f5990d32a8adbc9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=394b0963ad0828345e14c3be23cb7790, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c739eeba765df343af4ef63444655c9f, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,473 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b703e982cd6bbf72ec2f771b120c649, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,473 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b076d46dbfff2e1f3c89ad5c3f814d, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,473 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fbe24cda440bbf727cd581ab33ad7a9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,473 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d7c21378c2e7788ac010db26909da83, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,473 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b81a9be48fbe50dde523592a87a4611, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,473 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4795f188f4931ea02df11d7474e4daa4, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,474 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e38ac98b32f350d84df37b6915c56a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,474 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7b0558136890d45b56827e5af943cfa, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,474 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56e168ba1a3103978bee5b6c43feba36, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,474 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec76766d593d312811f043efb129a5a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,474 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0178ff26f4e2d0ab40d50fe6fcd026be, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,475 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fde2b7fbe9fa7c8a87ab544f5542d82a, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,475 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e4348dabb8addd0673ec5266827d8b4d, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,475 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-12T18:14:50,475 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731435350475 2024-11-12T18:14:50,475 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731435410475 2024-11-12T18:14:50,475 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 11 msec 2024-11-12T18:14:50,475 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T18:14:50,475 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,475 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T18:14:50,475 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T18:14:50,476 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T18:14:50,476 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T18:14:50,476 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T18:14:50,476 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T18:14:50,477 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T18:14:50,478 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T18:14:50,479 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T18:14:50,479 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T18:14:50,479 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,479 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,479 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,479 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,479 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,479 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,479 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,479 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T18:14:50,479 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T18:14:50,479 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,479 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,479 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T18:14:50,479 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T18:14:50,480 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6e290ad15dec09a0cb11782988af7c4c, REOPEN/MOVE 2024-11-12T18:14:50,480 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true 2024-11-12T18:14:50,480 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,480 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T18:14:50,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN 2024-11-12T18:14:50,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN 2024-11-12T18:14:50,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN 2024-11-12T18:14:50,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN 2024-11-12T18:14:50,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN 2024-11-12T18:14:50,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN 2024-11-12T18:14:50,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN 2024-11-12T18:14:50,481 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,481 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,481 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,481 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,481 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,481 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,481 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,481 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,481 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN 2024-11-12T18:14:50,481 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,481 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN 2024-11-12T18:14:50,481 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,481 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN 2024-11-12T18:14:50,481 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,481 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN 2024-11-12T18:14:50,481 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,481 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN 2024-11-12T18:14:50,481 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,481 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN 2024-11-12T18:14:50,481 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,481 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN 2024-11-12T18:14:50,481 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-12T18:14:50,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=65 2024-11-12T18:14:50,482 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9911683f163c,34847,1731435282512, table=hbase:meta, region=1588230740 2024-11-12T18:14:50,483 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9911683f163c,34847,1731435282512, state=OPEN 2024-11-12T18:14:50,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,485 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,485 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,485 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,486 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN to state=OFFLINE, location=null, table=test, region=26fc65f8daf71d5831131b1bc6746a63 to restore RIT 2024-11-12T18:14:50,486 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN to state=OFFLINE, location=null, table=test, region=56e168ba1a3103978bee5b6c43feba36 to restore RIT 2024-11-12T18:14:50,486 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN to state=OFFLINE, location=null, table=test, region=1b81a9be48fbe50dde523592a87a4611 to restore RIT 2024-11-12T18:14:50,486 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN to state=OFFLINE, location=null, table=test, region=4b703e982cd6bbf72ec2f771b120c649 to restore RIT 2024-11-12T18:14:50,486 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN to state=OFFLINE, location=null, table=test, region=d59e820eb0eef2601f5990d32a8adbc9 to restore RIT 2024-11-12T18:14:50,486 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN to state=OFFLINE, location=null, table=test, region=9fbe24cda440bbf727cd581ab33ad7a9 to restore RIT 2024-11-12T18:14:50,486 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2248a0d1296d1dbe4f8c9805798db03c, ASSIGN to state=OFFLINE, location=null, table=test, region=2248a0d1296d1dbe4f8c9805798db03c to restore RIT 2024-11-12T18:14:50,487 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731435320487 2024-11-12T18:14:50,487 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-12T18:14:50,487 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,489 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T18:14:50,490 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T18:14:50,490 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN 2024-11-12T18:14:50,490 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,490 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,493 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dcfe82d27e00f564e4ad71ea2e78f5c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,493 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2248a0d1296d1dbe4f8c9805798db03c, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,493 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a62a194de5221e9f84b2cfd2e83a46a, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,494 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e290ad15dec09a0cb11782988af7c4c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=5 2024-11-12T18:14:50,494 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0b5ce997df05cef2b0f3fd25e49fafc, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,494 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4aff96e33c9923b5d2448f72171fd310, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,494 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26fc65f8daf71d5831131b1bc6746a63, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,494 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dff0a9c660bd5084b34753ea40ea3278, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,494 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96e9ca1050c6b52ea6556fdb4b5d5f39, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,494 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a6d3052a2b28af2dab58e9247bb22b7, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,494 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d59e820eb0eef2601f5990d32a8adbc9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,495 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=394b0963ad0828345e14c3be23cb7790, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,495 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c739eeba765df343af4ef63444655c9f, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,495 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b703e982cd6bbf72ec2f771b120c649, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,495 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b076d46dbfff2e1f3c89ad5c3f814d, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,495 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fbe24cda440bbf727cd581ab33ad7a9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,495 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d7c21378c2e7788ac010db26909da83, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,496 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b81a9be48fbe50dde523592a87a4611, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,496 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4795f188f4931ea02df11d7474e4daa4, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,496 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e38ac98b32f350d84df37b6915c56a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,496 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7b0558136890d45b56827e5af943cfa, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,496 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56e168ba1a3103978bee5b6c43feba36, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec76766d593d312811f043efb129a5a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0178ff26f4e2d0ab40d50fe6fcd026be, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fde2b7fbe9fa7c8a87ab544f5542d82a, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e4348dabb8addd0673ec5266827d8b4d, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,497 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-12T18:14:50,497 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731435350497 2024-11-12T18:14:50,497 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731435410497 2024-11-12T18:14:50,497 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-12T18:14:50,498 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T18:14:50,498 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,498 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T18:14:50,498 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T18:14:50,498 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T18:14:50,498 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T18:14:50,498 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T18:14:50,498 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T18:14:50,499 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T18:14:50,502 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T18:14:50,502 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T18:14:50,502 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T18:14:50,502 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,502 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,502 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,502 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,502 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,502 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,502 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T18:14:50,502 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T18:14:50,502 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,502 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,502 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T18:14:50,503 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T18:14:50,503 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6e290ad15dec09a0cb11782988af7c4c, REOPEN/MOVE 2024-11-12T18:14:50,504 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36779 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,505 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true 2024-11-12T18:14:50,505 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,505 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,505 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T18:14:50,505 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,505 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,505 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,505 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN 2024-11-12T18:14:50,505 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN 2024-11-12T18:14:50,505 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN 2024-11-12T18:14:50,505 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN 2024-11-12T18:14:50,505 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN 2024-11-12T18:14:50,505 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN 2024-11-12T18:14:50,505 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,505 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,505 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,505 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,505 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,505 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,506 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,506 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,506 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN 2024-11-12T18:14:50,506 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,506 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN 2024-11-12T18:14:50,506 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,506 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN 2024-11-12T18:14:50,506 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,506 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN 2024-11-12T18:14:50,506 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,506 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN 2024-11-12T18:14:50,506 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,506 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN 2024-11-12T18:14:50,506 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-12T18:14:50,507 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=65 2024-11-12T18:14:50,507 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9911683f163c,34847,1731435282512, table=hbase:meta, region=1588230740 2024-11-12T18:14:50,507 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9911683f163c,34847,1731435282512, state=OPEN 2024-11-12T18:14:50,509 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36779 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,509 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,509 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,509 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,510 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN to state=OFFLINE, location=null, table=test, region=26fc65f8daf71d5831131b1bc6746a63 to restore RIT 2024-11-12T18:14:50,510 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN to state=OFFLINE, location=null, table=test, region=56e168ba1a3103978bee5b6c43feba36 to restore RIT 2024-11-12T18:14:50,510 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN to state=OFFLINE, location=null, table=test, region=1b81a9be48fbe50dde523592a87a4611 to restore RIT 2024-11-12T18:14:50,511 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN to state=OFFLINE, location=null, table=test, region=4b703e982cd6bbf72ec2f771b120c649 to restore RIT 2024-11-12T18:14:50,511 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN to state=OFFLINE, location=null, table=test, region=d59e820eb0eef2601f5990d32a8adbc9 to restore RIT 2024-11-12T18:14:50,511 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9fbe24cda440bbf727cd581ab33ad7a9, ASSIGN to state=OFFLINE, location=null, table=test, region=9fbe24cda440bbf727cd581ab33ad7a9 to restore RIT 2024-11-12T18:14:50,511 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731435320511 2024-11-12T18:14:50,511 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-12T18:14:50,511 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,513 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T18:14:50,516 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T18:14:50,516 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN 2024-11-12T18:14:50,516 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,516 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,516 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dcfe82d27e00f564e4ad71ea2e78f5c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,517 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2248a0d1296d1dbe4f8c9805798db03c, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,517 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a62a194de5221e9f84b2cfd2e83a46a, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,517 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e290ad15dec09a0cb11782988af7c4c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=5 2024-11-12T18:14:50,517 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0b5ce997df05cef2b0f3fd25e49fafc, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,517 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4aff96e33c9923b5d2448f72171fd310, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,518 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26fc65f8daf71d5831131b1bc6746a63, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,518 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dff0a9c660bd5084b34753ea40ea3278, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,518 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96e9ca1050c6b52ea6556fdb4b5d5f39, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,518 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a6d3052a2b28af2dab58e9247bb22b7, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,518 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d59e820eb0eef2601f5990d32a8adbc9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,518 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=394b0963ad0828345e14c3be23cb7790, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,519 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c739eeba765df343af4ef63444655c9f, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,519 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b703e982cd6bbf72ec2f771b120c649, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,519 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b076d46dbfff2e1f3c89ad5c3f814d, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,519 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fbe24cda440bbf727cd581ab33ad7a9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,519 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d7c21378c2e7788ac010db26909da83, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,519 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b81a9be48fbe50dde523592a87a4611, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4795f188f4931ea02df11d7474e4daa4, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e38ac98b32f350d84df37b6915c56a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7b0558136890d45b56827e5af943cfa, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56e168ba1a3103978bee5b6c43feba36, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec76766d593d312811f043efb129a5a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0178ff26f4e2d0ab40d50fe6fcd026be, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fde2b7fbe9fa7c8a87ab544f5542d82a, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e4348dabb8addd0673ec5266827d8b4d, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,521 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-12T18:14:50,521 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731435350521 2024-11-12T18:14:50,521 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731435410521 2024-11-12T18:14:50,521 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-12T18:14:50,521 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T18:14:50,521 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,521 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T18:14:50,521 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T18:14:50,521 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T18:14:50,522 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T18:14:50,522 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T18:14:50,522 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T18:14:50,523 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T18:14:50,525 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T18:14:50,525 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T18:14:50,525 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T18:14:50,525 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,525 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,525 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,525 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,525 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,525 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T18:14:50,525 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T18:14:50,525 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,525 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,525 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T18:14:50,526 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T18:14:50,526 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6e290ad15dec09a0cb11782988af7c4c, REOPEN/MOVE 2024-11-12T18:14:50,527 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true 2024-11-12T18:14:50,527 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,527 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,527 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T18:14:50,527 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,527 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,527 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN 2024-11-12T18:14:50,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN 2024-11-12T18:14:50,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN 2024-11-12T18:14:50,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN 2024-11-12T18:14:50,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN 2024-11-12T18:14:50,528 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,528 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,528 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,528 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,528 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,528 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,528 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,528 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,528 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN 2024-11-12T18:14:50,528 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,528 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN 2024-11-12T18:14:50,528 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,528 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN 2024-11-12T18:14:50,528 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,528 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN 2024-11-12T18:14:50,528 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,528 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN 2024-11-12T18:14:50,529 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-12T18:14:50,530 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=65 2024-11-12T18:14:50,530 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9911683f163c,34847,1731435282512, table=hbase:meta, region=1588230740 2024-11-12T18:14:50,530 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9911683f163c,34847,1731435282512, state=OPEN 2024-11-12T18:14:50,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,532 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,532 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,532 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,534 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN to state=OFFLINE, location=null, table=test, region=26fc65f8daf71d5831131b1bc6746a63 to restore RIT 2024-11-12T18:14:50,534 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN to state=OFFLINE, location=null, table=test, region=56e168ba1a3103978bee5b6c43feba36 to restore RIT 2024-11-12T18:14:50,534 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN to state=OFFLINE, location=null, table=test, region=1b81a9be48fbe50dde523592a87a4611 to restore RIT 2024-11-12T18:14:50,534 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN to state=OFFLINE, location=null, table=test, region=4b703e982cd6bbf72ec2f771b120c649 to restore RIT 2024-11-12T18:14:50,534 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d59e820eb0eef2601f5990d32a8adbc9, ASSIGN to state=OFFLINE, location=null, table=test, region=d59e820eb0eef2601f5990d32a8adbc9 to restore RIT 2024-11-12T18:14:50,535 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731435320535 2024-11-12T18:14:50,535 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-12T18:14:50,535 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,536 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T18:14:50,538 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T18:14:50,538 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN 2024-11-12T18:14:50,538 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,538 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dcfe82d27e00f564e4ad71ea2e78f5c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2248a0d1296d1dbe4f8c9805798db03c, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a62a194de5221e9f84b2cfd2e83a46a, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e290ad15dec09a0cb11782988af7c4c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=5 2024-11-12T18:14:50,542 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0b5ce997df05cef2b0f3fd25e49fafc, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,542 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4aff96e33c9923b5d2448f72171fd310, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,542 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26fc65f8daf71d5831131b1bc6746a63, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,542 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dff0a9c660bd5084b34753ea40ea3278, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,542 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96e9ca1050c6b52ea6556fdb4b5d5f39, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,543 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a6d3052a2b28af2dab58e9247bb22b7, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,543 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d59e820eb0eef2601f5990d32a8adbc9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,543 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=394b0963ad0828345e14c3be23cb7790, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,543 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c739eeba765df343af4ef63444655c9f, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,543 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b703e982cd6bbf72ec2f771b120c649, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,544 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b076d46dbfff2e1f3c89ad5c3f814d, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,544 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fbe24cda440bbf727cd581ab33ad7a9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,544 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d7c21378c2e7788ac010db26909da83, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,544 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b81a9be48fbe50dde523592a87a4611, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,544 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4795f188f4931ea02df11d7474e4daa4, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,544 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e38ac98b32f350d84df37b6915c56a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,545 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7b0558136890d45b56827e5af943cfa, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,545 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56e168ba1a3103978bee5b6c43feba36, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,545 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec76766d593d312811f043efb129a5a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,545 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0178ff26f4e2d0ab40d50fe6fcd026be, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,545 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fde2b7fbe9fa7c8a87ab544f5542d82a, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,546 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e4348dabb8addd0673ec5266827d8b4d, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,546 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-12T18:14:50,546 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731435350546 2024-11-12T18:14:50,546 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731435410546 2024-11-12T18:14:50,546 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 11 msec 2024-11-12T18:14:50,546 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T18:14:50,546 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,546 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T18:14:50,546 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T18:14:50,546 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T18:14:50,547 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T18:14:50,547 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T18:14:50,547 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T18:14:50,548 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T18:14:50,550 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T18:14:50,550 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T18:14:50,550 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T18:14:50,550 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,550 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,550 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,550 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,550 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T18:14:50,550 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T18:14:50,550 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,550 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,550 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T18:14:50,551 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T18:14:50,551 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6e290ad15dec09a0cb11782988af7c4c, REOPEN/MOVE 2024-11-12T18:14:50,553 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true 2024-11-12T18:14:50,553 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,553 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,553 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T18:14:50,553 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,553 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,553 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,553 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN 2024-11-12T18:14:50,553 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN 2024-11-12T18:14:50,553 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN 2024-11-12T18:14:50,553 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN 2024-11-12T18:14:50,554 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,554 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,554 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,554 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,554 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,554 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,554 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,554 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,554 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN 2024-11-12T18:14:50,554 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,554 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN 2024-11-12T18:14:50,554 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,554 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN 2024-11-12T18:14:50,554 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,554 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN 2024-11-12T18:14:50,554 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-12T18:14:50,555 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=65 2024-11-12T18:14:50,555 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9911683f163c,34847,1731435282512, table=hbase:meta, region=1588230740 2024-11-12T18:14:50,556 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9911683f163c,34847,1731435282512, state=OPEN 2024-11-12T18:14:50,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,558 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,558 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,558 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,559 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN to state=OFFLINE, location=null, table=test, region=26fc65f8daf71d5831131b1bc6746a63 to restore RIT 2024-11-12T18:14:50,559 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN to state=OFFLINE, location=null, table=test, region=56e168ba1a3103978bee5b6c43feba36 to restore RIT 2024-11-12T18:14:50,559 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN to state=OFFLINE, location=null, table=test, region=1b81a9be48fbe50dde523592a87a4611 to restore RIT 2024-11-12T18:14:50,559 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4b703e982cd6bbf72ec2f771b120c649, ASSIGN to state=OFFLINE, location=null, table=test, region=4b703e982cd6bbf72ec2f771b120c649 to restore RIT 2024-11-12T18:14:50,559 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731435320559 2024-11-12T18:14:50,559 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-12T18:14:50,560 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,561 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T18:14:50,562 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T18:14:50,562 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN 2024-11-12T18:14:50,562 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,562 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,564 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dcfe82d27e00f564e4ad71ea2e78f5c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,564 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2248a0d1296d1dbe4f8c9805798db03c, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,565 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a62a194de5221e9f84b2cfd2e83a46a, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,565 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e290ad15dec09a0cb11782988af7c4c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=5 2024-11-12T18:14:50,565 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0b5ce997df05cef2b0f3fd25e49fafc, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,565 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4aff96e33c9923b5d2448f72171fd310, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,565 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26fc65f8daf71d5831131b1bc6746a63, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,565 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dff0a9c660bd5084b34753ea40ea3278, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,565 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96e9ca1050c6b52ea6556fdb4b5d5f39, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,565 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a6d3052a2b28af2dab58e9247bb22b7, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,565 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d59e820eb0eef2601f5990d32a8adbc9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,565 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=394b0963ad0828345e14c3be23cb7790, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,565 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c739eeba765df343af4ef63444655c9f, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,566 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b703e982cd6bbf72ec2f771b120c649, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,566 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b076d46dbfff2e1f3c89ad5c3f814d, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,566 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fbe24cda440bbf727cd581ab33ad7a9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,566 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d7c21378c2e7788ac010db26909da83, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,566 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b81a9be48fbe50dde523592a87a4611, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,566 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4795f188f4931ea02df11d7474e4daa4, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,566 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e38ac98b32f350d84df37b6915c56a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,566 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7b0558136890d45b56827e5af943cfa, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,566 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56e168ba1a3103978bee5b6c43feba36, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,566 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec76766d593d312811f043efb129a5a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,567 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0178ff26f4e2d0ab40d50fe6fcd026be, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,567 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fde2b7fbe9fa7c8a87ab544f5542d82a, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,567 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e4348dabb8addd0673ec5266827d8b4d, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,567 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-12T18:14:50,567 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731435350567 2024-11-12T18:14:50,567 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731435410567 2024-11-12T18:14:50,567 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-12T18:14:50,567 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T18:14:50,567 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,567 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T18:14:50,567 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T18:14:50,567 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T18:14:50,567 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T18:14:50,567 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T18:14:50,568 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T18:14:50,568 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T18:14:50,570 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T18:14:50,570 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T18:14:50,570 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T18:14:50,570 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,570 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,570 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,570 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T18:14:50,570 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T18:14:50,570 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,570 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,570 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T18:14:50,570 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T18:14:50,571 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6e290ad15dec09a0cb11782988af7c4c, REOPEN/MOVE 2024-11-12T18:14:50,571 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true 2024-11-12T18:14:50,571 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,571 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,571 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T18:14:50,571 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,571 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,571 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,571 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN 2024-11-12T18:14:50,571 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN 2024-11-12T18:14:50,571 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN 2024-11-12T18:14:50,572 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,572 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,572 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,572 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,572 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,572 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,572 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,572 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,572 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN 2024-11-12T18:14:50,572 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,572 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN 2024-11-12T18:14:50,572 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,572 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN 2024-11-12T18:14:50,572 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-12T18:14:50,573 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=65 2024-11-12T18:14:50,573 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9911683f163c,34847,1731435282512, table=hbase:meta, region=1588230740 2024-11-12T18:14:50,573 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9911683f163c,34847,1731435282512, state=OPEN 2024-11-12T18:14:50,575 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,575 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,575 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,575 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,575 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,575 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,577 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN to state=OFFLINE, location=null, table=test, region=26fc65f8daf71d5831131b1bc6746a63 to restore RIT 2024-11-12T18:14:50,577 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN to state=OFFLINE, location=null, table=test, region=56e168ba1a3103978bee5b6c43feba36 to restore RIT 2024-11-12T18:14:50,577 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1b81a9be48fbe50dde523592a87a4611, ASSIGN to state=OFFLINE, location=null, table=test, region=1b81a9be48fbe50dde523592a87a4611 to restore RIT 2024-11-12T18:14:50,579 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731435320579 2024-11-12T18:14:50,579 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-12T18:14:50,579 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,580 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T18:14:50,582 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T18:14:50,582 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN 2024-11-12T18:14:50,582 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,582 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dcfe82d27e00f564e4ad71ea2e78f5c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2248a0d1296d1dbe4f8c9805798db03c, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a62a194de5221e9f84b2cfd2e83a46a, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e290ad15dec09a0cb11782988af7c4c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=5 2024-11-12T18:14:50,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0b5ce997df05cef2b0f3fd25e49fafc, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4aff96e33c9923b5d2448f72171fd310, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26fc65f8daf71d5831131b1bc6746a63, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dff0a9c660bd5084b34753ea40ea3278, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96e9ca1050c6b52ea6556fdb4b5d5f39, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a6d3052a2b28af2dab58e9247bb22b7, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d59e820eb0eef2601f5990d32a8adbc9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=394b0963ad0828345e14c3be23cb7790, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c739eeba765df343af4ef63444655c9f, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b703e982cd6bbf72ec2f771b120c649, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b076d46dbfff2e1f3c89ad5c3f814d, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fbe24cda440bbf727cd581ab33ad7a9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d7c21378c2e7788ac010db26909da83, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b81a9be48fbe50dde523592a87a4611, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4795f188f4931ea02df11d7474e4daa4, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e38ac98b32f350d84df37b6915c56a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7b0558136890d45b56827e5af943cfa, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56e168ba1a3103978bee5b6c43feba36, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec76766d593d312811f043efb129a5a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0178ff26f4e2d0ab40d50fe6fcd026be, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fde2b7fbe9fa7c8a87ab544f5542d82a, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e4348dabb8addd0673ec5266827d8b4d, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,588 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-12T18:14:50,588 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731435350588 2024-11-12T18:14:50,588 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731435410588 2024-11-12T18:14:50,588 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-12T18:14:50,589 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T18:14:50,589 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,589 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T18:14:50,589 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T18:14:50,589 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T18:14:50,589 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T18:14:50,589 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T18:14:50,589 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T18:14:50,590 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T18:14:50,592 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T18:14:50,592 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T18:14:50,592 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T18:14:50,592 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,592 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,592 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T18:14:50,592 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T18:14:50,592 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,592 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,592 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T18:14:50,593 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T18:14:50,593 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6e290ad15dec09a0cb11782988af7c4c, REOPEN/MOVE 2024-11-12T18:14:50,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true 2024-11-12T18:14:50,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T18:14:50,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN 2024-11-12T18:14:50,595 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN 2024-11-12T18:14:50,595 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,595 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,595 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,595 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,595 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,595 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,595 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,595 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,595 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN 2024-11-12T18:14:50,595 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,595 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN 2024-11-12T18:14:50,595 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-12T18:14:50,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=65 2024-11-12T18:14:50,596 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9911683f163c,34847,1731435282512, table=hbase:meta, region=1588230740 2024-11-12T18:14:50,596 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9911683f163c,34847,1731435282512, state=OPEN 2024-11-12T18:14:50,601 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,601 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,602 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,602 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,602 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,603 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN to state=OFFLINE, location=null, table=test, region=26fc65f8daf71d5831131b1bc6746a63 to restore RIT 2024-11-12T18:14:50,603 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=56e168ba1a3103978bee5b6c43feba36, ASSIGN to state=OFFLINE, location=null, table=test, region=56e168ba1a3103978bee5b6c43feba36 to restore RIT 2024-11-12T18:14:50,603 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731435320603 2024-11-12T18:14:50,603 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-12T18:14:50,603 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,605 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T18:14:50,606 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T18:14:50,606 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN 2024-11-12T18:14:50,606 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,606 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dcfe82d27e00f564e4ad71ea2e78f5c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2248a0d1296d1dbe4f8c9805798db03c, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,610 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a62a194de5221e9f84b2cfd2e83a46a, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,610 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e290ad15dec09a0cb11782988af7c4c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=5 2024-11-12T18:14:50,610 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0b5ce997df05cef2b0f3fd25e49fafc, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,610 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4aff96e33c9923b5d2448f72171fd310, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,610 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26fc65f8daf71d5831131b1bc6746a63, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,610 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dff0a9c660bd5084b34753ea40ea3278, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,610 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96e9ca1050c6b52ea6556fdb4b5d5f39, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,610 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a6d3052a2b28af2dab58e9247bb22b7, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,610 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d59e820eb0eef2601f5990d32a8adbc9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,610 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=394b0963ad0828345e14c3be23cb7790, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,610 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c739eeba765df343af4ef63444655c9f, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,610 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b703e982cd6bbf72ec2f771b120c649, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,611 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b076d46dbfff2e1f3c89ad5c3f814d, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,611 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fbe24cda440bbf727cd581ab33ad7a9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,611 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d7c21378c2e7788ac010db26909da83, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,611 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b81a9be48fbe50dde523592a87a4611, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,611 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4795f188f4931ea02df11d7474e4daa4, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,611 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e38ac98b32f350d84df37b6915c56a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,611 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7b0558136890d45b56827e5af943cfa, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,611 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56e168ba1a3103978bee5b6c43feba36, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,611 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec76766d593d312811f043efb129a5a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,612 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0178ff26f4e2d0ab40d50fe6fcd026be, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,612 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fde2b7fbe9fa7c8a87ab544f5542d82a, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,612 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e4348dabb8addd0673ec5266827d8b4d, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,612 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-12T18:14:50,612 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731435350612 2024-11-12T18:14:50,612 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731435410612 2024-11-12T18:14:50,612 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-12T18:14:50,612 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T18:14:50,612 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,612 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T18:14:50,612 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T18:14:50,613 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T18:14:50,613 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T18:14:50,613 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T18:14:50,613 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T18:14:50,614 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T18:14:50,616 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T18:14:50,616 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T18:14:50,616 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T18:14:50,616 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,616 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T18:14:50,616 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T18:14:50,616 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,616 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,616 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T18:14:50,617 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T18:14:50,617 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6e290ad15dec09a0cb11782988af7c4c, REOPEN/MOVE 2024-11-12T18:14:50,618 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true 2024-11-12T18:14:50,618 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,618 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,618 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T18:14:50,618 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,618 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,618 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,618 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN 2024-11-12T18:14:50,618 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,618 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,618 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,618 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,618 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,619 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,619 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,619 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,619 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN 2024-11-12T18:14:50,619 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-12T18:14:50,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=65 2024-11-12T18:14:50,620 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9911683f163c,34847,1731435282512, table=hbase:meta, region=1588230740 2024-11-12T18:14:50,620 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9911683f163c,34847,1731435282512, state=OPEN 2024-11-12T18:14:50,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,622 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,622 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,622 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,622 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,622 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=26fc65f8daf71d5831131b1bc6746a63, ASSIGN to state=OFFLINE, location=null, table=test, region=26fc65f8daf71d5831131b1bc6746a63 to restore RIT 2024-11-12T18:14:50,627 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731435320627 2024-11-12T18:14:50,628 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-12T18:14:50,628 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,629 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T18:14:50,630 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T18:14:50,630 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,630 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,631 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,632 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dcfe82d27e00f564e4ad71ea2e78f5c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,633 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2248a0d1296d1dbe4f8c9805798db03c, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,633 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a62a194de5221e9f84b2cfd2e83a46a, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,633 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e290ad15dec09a0cb11782988af7c4c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=5 2024-11-12T18:14:50,633 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0b5ce997df05cef2b0f3fd25e49fafc, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,633 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4aff96e33c9923b5d2448f72171fd310, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,633 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26fc65f8daf71d5831131b1bc6746a63, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,633 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dff0a9c660bd5084b34753ea40ea3278, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,633 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96e9ca1050c6b52ea6556fdb4b5d5f39, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,634 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a6d3052a2b28af2dab58e9247bb22b7, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,634 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d59e820eb0eef2601f5990d32a8adbc9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,634 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=394b0963ad0828345e14c3be23cb7790, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,634 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c739eeba765df343af4ef63444655c9f, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,634 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b703e982cd6bbf72ec2f771b120c649, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,634 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b076d46dbfff2e1f3c89ad5c3f814d, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,634 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fbe24cda440bbf727cd581ab33ad7a9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,634 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d7c21378c2e7788ac010db26909da83, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,634 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b81a9be48fbe50dde523592a87a4611, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,634 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4795f188f4931ea02df11d7474e4daa4, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,634 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e38ac98b32f350d84df37b6915c56a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,634 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7b0558136890d45b56827e5af943cfa, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,635 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56e168ba1a3103978bee5b6c43feba36, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,635 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec76766d593d312811f043efb129a5a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,635 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0178ff26f4e2d0ab40d50fe6fcd026be, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,635 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fde2b7fbe9fa7c8a87ab544f5542d82a, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,635 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e4348dabb8addd0673ec5266827d8b4d, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,635 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-12T18:14:50,635 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731435350635 2024-11-12T18:14:50,635 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731435410635 2024-11-12T18:14:50,635 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-12T18:14:50,635 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T18:14:50,635 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,636 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T18:14:50,636 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T18:14:50,636 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T18:14:50,636 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T18:14:50,636 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T18:14:50,636 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T18:14:50,637 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T18:14:50,638 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T18:14:50,639 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T18:14:50,639 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T18:14:50,639 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T18:14:50,639 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T18:14:50,639 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,639 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,639 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T18:14:50,639 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T18:14:50,640 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6e290ad15dec09a0cb11782988af7c4c, REOPEN/MOVE 2024-11-12T18:14:50,640 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true 2024-11-12T18:14:50,640 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,641 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,641 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T18:14:50,641 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,641 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,641 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,641 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,641 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,641 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,641 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,641 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,641 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,641 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,641 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-12T18:14:50,642 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=65 2024-11-12T18:14:50,642 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9911683f163c,34847,1731435282512, table=hbase:meta, region=1588230740 2024-11-12T18:14:50,642 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9911683f163c,34847,1731435282512, state=OPEN 2024-11-12T18:14:50,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,644 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,644 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,644 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,645 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731435320645 2024-11-12T18:14:50,645 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-12T18:14:50,645 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,647 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T18:14:50,649 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T18:14:50,649 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,649 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,649 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,650 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dcfe82d27e00f564e4ad71ea2e78f5c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,651 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2248a0d1296d1dbe4f8c9805798db03c, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,651 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a62a194de5221e9f84b2cfd2e83a46a, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,651 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e290ad15dec09a0cb11782988af7c4c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=5 2024-11-12T18:14:50,651 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0b5ce997df05cef2b0f3fd25e49fafc, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,651 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4aff96e33c9923b5d2448f72171fd310, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,651 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26fc65f8daf71d5831131b1bc6746a63, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,651 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dff0a9c660bd5084b34753ea40ea3278, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,651 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96e9ca1050c6b52ea6556fdb4b5d5f39, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,651 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a6d3052a2b28af2dab58e9247bb22b7, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,651 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d59e820eb0eef2601f5990d32a8adbc9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,652 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=394b0963ad0828345e14c3be23cb7790, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,652 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c739eeba765df343af4ef63444655c9f, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,652 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b703e982cd6bbf72ec2f771b120c649, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,652 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b076d46dbfff2e1f3c89ad5c3f814d, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,652 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fbe24cda440bbf727cd581ab33ad7a9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,652 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d7c21378c2e7788ac010db26909da83, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,652 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b81a9be48fbe50dde523592a87a4611, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,652 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4795f188f4931ea02df11d7474e4daa4, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,652 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e38ac98b32f350d84df37b6915c56a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,652 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7b0558136890d45b56827e5af943cfa, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,653 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56e168ba1a3103978bee5b6c43feba36, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,653 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec76766d593d312811f043efb129a5a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,653 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0178ff26f4e2d0ab40d50fe6fcd026be, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,653 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fde2b7fbe9fa7c8a87ab544f5542d82a, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,653 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e4348dabb8addd0673ec5266827d8b4d, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,653 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-12T18:14:50,653 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731435350653 2024-11-12T18:14:50,653 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731435410653 2024-11-12T18:14:50,653 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-12T18:14:50,653 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T18:14:50,653 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,653 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T18:14:50,653 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T18:14:50,653 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T18:14:50,654 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T18:14:50,654 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T18:14:50,654 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T18:14:50,654 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T18:14:50,656 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T18:14:50,656 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T18:14:50,656 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T18:14:50,656 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T18:14:50,656 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,656 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,656 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T18:14:50,657 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T18:14:50,657 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6e290ad15dec09a0cb11782988af7c4c, REOPEN/MOVE 2024-11-12T18:14:50,657 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true 2024-11-12T18:14:50,657 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,657 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,657 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T18:14:50,657 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,657 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,658 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,658 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,658 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,658 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,658 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,658 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.1731435284060, worker=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,658 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-12T18:14:50,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=65 2024-11-12T18:14:50,659 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9911683f163c,34847,1731435282512, table=hbase:meta, region=1588230740 2024-11-12T18:14:50,659 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9911683f163c,34847,1731435282512, state=OPEN 2024-11-12T18:14:50,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,661 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,661 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,661 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,662 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731435320662 2024-11-12T18:14:50,662 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-12T18:14:50,662 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,663 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T18:14:50,665 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T18:14:50,665 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,665 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,665 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,34847,1731435282512 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,667 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dcfe82d27e00f564e4ad71ea2e78f5c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,667 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2248a0d1296d1dbe4f8c9805798db03c, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,667 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a62a194de5221e9f84b2cfd2e83a46a, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,667 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e290ad15dec09a0cb11782988af7c4c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=5 2024-11-12T18:14:50,667 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0b5ce997df05cef2b0f3fd25e49fafc, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4aff96e33c9923b5d2448f72171fd310, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26fc65f8daf71d5831131b1bc6746a63, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dff0a9c660bd5084b34753ea40ea3278, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96e9ca1050c6b52ea6556fdb4b5d5f39, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a6d3052a2b28af2dab58e9247bb22b7, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d59e820eb0eef2601f5990d32a8adbc9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=394b0963ad0828345e14c3be23cb7790, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c739eeba765df343af4ef63444655c9f, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b703e982cd6bbf72ec2f771b120c649, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b076d46dbfff2e1f3c89ad5c3f814d, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fbe24cda440bbf727cd581ab33ad7a9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d7c21378c2e7788ac010db26909da83, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b81a9be48fbe50dde523592a87a4611, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4795f188f4931ea02df11d7474e4daa4, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e38ac98b32f350d84df37b6915c56a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7b0558136890d45b56827e5af943cfa, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56e168ba1a3103978bee5b6c43feba36, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec76766d593d312811f043efb129a5a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0178ff26f4e2d0ab40d50fe6fcd026be, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fde2b7fbe9fa7c8a87ab544f5542d82a, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e4348dabb8addd0673ec5266827d8b4d, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,670 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-12T18:14:50,670 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731435350670 2024-11-12T18:14:50,670 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731435410670 2024-11-12T18:14:50,670 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-12T18:14:50,670 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T18:14:50,670 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,670 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T18:14:50,670 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T18:14:50,670 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T18:14:50,670 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T18:14:50,671 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T18:14:50,671 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T18:14:50,672 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T18:14:50,673 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T18:14:50,673 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T18:14:50,673 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T18:14:50,673 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,673 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T18:14:50,673 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T18:14:50,674 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T18:14:50,674 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6e290ad15dec09a0cb11782988af7c4c, REOPEN/MOVE 2024-11-12T18:14:50,674 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true 2024-11-12T18:14:50,674 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T18:14:50,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,34847,1731435282512 2024-11-12T18:14:50,675 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,675 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,675 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,675 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,675 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,34847,1731435282512 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,675 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-12T18:14:50,676 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=65 2024-11-12T18:14:50,676 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9911683f163c,34847,1731435282512, table=hbase:meta, region=1588230740 2024-11-12T18:14:50,676 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9911683f163c,34847,1731435282512, state=OPEN 2024-11-12T18:14:50,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,678 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,678 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,679 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,680 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731435320680 2024-11-12T18:14:50,681 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-12T18:14:50,681 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,682 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T18:14:50,683 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T18:14:50,683 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T18:14:50,683 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,683 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,686 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dcfe82d27e00f564e4ad71ea2e78f5c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,686 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2248a0d1296d1dbe4f8c9805798db03c, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,686 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a62a194de5221e9f84b2cfd2e83a46a, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,686 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e290ad15dec09a0cb11782988af7c4c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=5 2024-11-12T18:14:50,686 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0b5ce997df05cef2b0f3fd25e49fafc, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,686 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4aff96e33c9923b5d2448f72171fd310, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,686 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26fc65f8daf71d5831131b1bc6746a63, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,686 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dff0a9c660bd5084b34753ea40ea3278, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,686 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96e9ca1050c6b52ea6556fdb4b5d5f39, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,686 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a6d3052a2b28af2dab58e9247bb22b7, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,686 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d59e820eb0eef2601f5990d32a8adbc9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,686 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=394b0963ad0828345e14c3be23cb7790, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,687 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c739eeba765df343af4ef63444655c9f, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,687 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b703e982cd6bbf72ec2f771b120c649, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,687 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b076d46dbfff2e1f3c89ad5c3f814d, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,687 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fbe24cda440bbf727cd581ab33ad7a9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,687 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d7c21378c2e7788ac010db26909da83, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,687 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b81a9be48fbe50dde523592a87a4611, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,687 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4795f188f4931ea02df11d7474e4daa4, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,687 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e38ac98b32f350d84df37b6915c56a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,687 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7b0558136890d45b56827e5af943cfa, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,687 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56e168ba1a3103978bee5b6c43feba36, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,687 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec76766d593d312811f043efb129a5a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,687 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0178ff26f4e2d0ab40d50fe6fcd026be, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,687 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fde2b7fbe9fa7c8a87ab544f5542d82a, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,688 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e4348dabb8addd0673ec5266827d8b4d, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,688 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-12T18:14:50,688 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731435350688 2024-11-12T18:14:50,688 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731435410688 2024-11-12T18:14:50,688 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-12T18:14:50,688 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T18:14:50,688 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,688 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T18:14:50,688 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T18:14:50,688 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T18:14:50,688 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T18:14:50,688 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T18:14:50,688 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T18:14:50,689 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T18:14:50,691 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T18:14:50,691 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T18:14:50,691 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T18:14:50,691 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T18:14:50,691 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T18:14:50,692 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T18:14:50,692 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6e290ad15dec09a0cb11782988af7c4c, REOPEN/MOVE 2024-11-12T18:14:50,692 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true 2024-11-12T18:14:50,693 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,693 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,693 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T18:14:50,693 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,693 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,693 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,693 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,693 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-12T18:14:50,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=65 2024-11-12T18:14:50,694 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9911683f163c,34847,1731435282512, table=hbase:meta, region=1588230740 2024-11-12T18:14:50,694 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9911683f163c,34847,1731435282512, state=OPEN 2024-11-12T18:14:50,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,696 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,696 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,696 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,697 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731435320697 2024-11-12T18:14:50,697 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-12T18:14:50,697 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,699 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T18:14:50,700 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T18:14:50,700 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,701 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dcfe82d27e00f564e4ad71ea2e78f5c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2248a0d1296d1dbe4f8c9805798db03c, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a62a194de5221e9f84b2cfd2e83a46a, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e290ad15dec09a0cb11782988af7c4c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=5 2024-11-12T18:14:50,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0b5ce997df05cef2b0f3fd25e49fafc, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4aff96e33c9923b5d2448f72171fd310, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26fc65f8daf71d5831131b1bc6746a63, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dff0a9c660bd5084b34753ea40ea3278, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96e9ca1050c6b52ea6556fdb4b5d5f39, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a6d3052a2b28af2dab58e9247bb22b7, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d59e820eb0eef2601f5990d32a8adbc9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=394b0963ad0828345e14c3be23cb7790, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c739eeba765df343af4ef63444655c9f, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b703e982cd6bbf72ec2f771b120c649, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b076d46dbfff2e1f3c89ad5c3f814d, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fbe24cda440bbf727cd581ab33ad7a9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d7c21378c2e7788ac010db26909da83, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b81a9be48fbe50dde523592a87a4611, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4795f188f4931ea02df11d7474e4daa4, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e38ac98b32f350d84df37b6915c56a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7b0558136890d45b56827e5af943cfa, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,704 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56e168ba1a3103978bee5b6c43feba36, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,704 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec76766d593d312811f043efb129a5a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,704 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0178ff26f4e2d0ab40d50fe6fcd026be, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,704 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fde2b7fbe9fa7c8a87ab544f5542d82a, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,704 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e4348dabb8addd0673ec5266827d8b4d, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,704 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-12T18:14:50,704 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731435350704 2024-11-12T18:14:50,704 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731435410704 2024-11-12T18:14:50,704 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 6 msec 2024-11-12T18:14:50,704 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T18:14:50,704 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,704 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T18:14:50,704 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T18:14:50,704 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T18:14:50,704 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T18:14:50,704 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T18:14:50,705 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T18:14:50,706 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T18:14:50,707 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T18:14:50,707 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T18:14:50,707 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T18:14:50,707 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T18:14:50,707 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T18:14:50,707 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6e290ad15dec09a0cb11782988af7c4c, REOPEN/MOVE 2024-11-12T18:14:50,708 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true 2024-11-12T18:14:50,708 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,708 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,708 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,708 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,708 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,708 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-12T18:14:50,709 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=65 2024-11-12T18:14:50,709 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9911683f163c,34847,1731435282512, table=hbase:meta, region=1588230740 2024-11-12T18:14:50,709 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9911683f163c,34847,1731435282512, state=OPEN 2024-11-12T18:14:50,710 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36779 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,711 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,711 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,711 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,712 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731435320712 2024-11-12T18:14:50,712 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-12T18:14:50,713 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,714 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T18:14:50,716 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T18:14:50,716 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,716 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,717 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dcfe82d27e00f564e4ad71ea2e78f5c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,717 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2248a0d1296d1dbe4f8c9805798db03c, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,717 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a62a194de5221e9f84b2cfd2e83a46a, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,717 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,717 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e290ad15dec09a0cb11782988af7c4c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=5 2024-11-12T18:14:50,718 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0b5ce997df05cef2b0f3fd25e49fafc, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,718 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4aff96e33c9923b5d2448f72171fd310, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,718 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26fc65f8daf71d5831131b1bc6746a63, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,718 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dff0a9c660bd5084b34753ea40ea3278, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,718 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96e9ca1050c6b52ea6556fdb4b5d5f39, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,718 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a6d3052a2b28af2dab58e9247bb22b7, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,718 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d59e820eb0eef2601f5990d32a8adbc9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,718 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=394b0963ad0828345e14c3be23cb7790, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,718 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c739eeba765df343af4ef63444655c9f, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,718 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b703e982cd6bbf72ec2f771b120c649, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,718 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b076d46dbfff2e1f3c89ad5c3f814d, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,718 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fbe24cda440bbf727cd581ab33ad7a9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,719 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d7c21378c2e7788ac010db26909da83, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,719 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b81a9be48fbe50dde523592a87a4611, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,719 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4795f188f4931ea02df11d7474e4daa4, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,719 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e38ac98b32f350d84df37b6915c56a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,719 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7b0558136890d45b56827e5af943cfa, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,719 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56e168ba1a3103978bee5b6c43feba36, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,719 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec76766d593d312811f043efb129a5a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,719 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0178ff26f4e2d0ab40d50fe6fcd026be, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,719 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fde2b7fbe9fa7c8a87ab544f5542d82a, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,719 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e4348dabb8addd0673ec5266827d8b4d, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,719 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-12T18:14:50,719 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731435350719 2024-11-12T18:14:50,719 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731435410719 2024-11-12T18:14:50,719 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 6 msec 2024-11-12T18:14:50,719 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T18:14:50,720 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,720 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T18:14:50,720 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T18:14:50,720 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T18:14:50,720 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T18:14:50,720 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T18:14:50,720 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T18:14:50,721 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T18:14:50,722 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T18:14:50,722 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T18:14:50,722 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T18:14:50,722 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T18:14:50,723 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6e290ad15dec09a0cb11782988af7c4c, REOPEN/MOVE 2024-11-12T18:14:50,723 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true 2024-11-12T18:14:50,723 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 2024-11-12T18:14:50,723 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,723 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 9911683f163c%2C40543%2C1731435282322.meta.1731435284522.meta, worker=9911683f163c,45243,1731435282573 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T18:14:50,723 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-12T18:14:50,724 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=65 2024-11-12T18:14:50,724 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9911683f163c,34847,1731435282512, table=hbase:meta, region=1588230740 2024-11-12T18:14:50,724 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9911683f163c,34847,1731435282512, state=OPEN 2024-11-12T18:14:50,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,726 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,726 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,726 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,727 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731435320727 2024-11-12T18:14:50,727 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-12T18:14:50,727 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,728 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T18:14:50,729 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T18:14:50,729 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true 2024-11-12T18:14:50,729 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,729 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dcfe82d27e00f564e4ad71ea2e78f5c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2248a0d1296d1dbe4f8c9805798db03c, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a62a194de5221e9f84b2cfd2e83a46a, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e290ad15dec09a0cb11782988af7c4c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=5 2024-11-12T18:14:50,732 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0b5ce997df05cef2b0f3fd25e49fafc, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,732 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4aff96e33c9923b5d2448f72171fd310, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,732 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26fc65f8daf71d5831131b1bc6746a63, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,732 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dff0a9c660bd5084b34753ea40ea3278, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,732 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96e9ca1050c6b52ea6556fdb4b5d5f39, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,732 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a6d3052a2b28af2dab58e9247bb22b7, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,732 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d59e820eb0eef2601f5990d32a8adbc9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=394b0963ad0828345e14c3be23cb7790, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c739eeba765df343af4ef63444655c9f, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b703e982cd6bbf72ec2f771b120c649, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b076d46dbfff2e1f3c89ad5c3f814d, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fbe24cda440bbf727cd581ab33ad7a9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d7c21378c2e7788ac010db26909da83, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b81a9be48fbe50dde523592a87a4611, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4795f188f4931ea02df11d7474e4daa4, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e38ac98b32f350d84df37b6915c56a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7b0558136890d45b56827e5af943cfa, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56e168ba1a3103978bee5b6c43feba36, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,734 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec76766d593d312811f043efb129a5a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,734 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0178ff26f4e2d0ab40d50fe6fcd026be, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,734 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fde2b7fbe9fa7c8a87ab544f5542d82a, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,734 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e4348dabb8addd0673ec5266827d8b4d, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,734 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-12T18:14:50,734 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731435350734 2024-11-12T18:14:50,734 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731435410734 2024-11-12T18:14:50,734 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-12T18:14:50,734 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T18:14:50,734 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,734 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T18:14:50,734 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T18:14:50,734 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T18:14:50,734 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T18:14:50,734 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T18:14:50,735 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T18:14:50,735 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T18:14:50,736 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T18:14:50,737 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T18:14:50,737 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T18:14:50,737 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6e290ad15dec09a0cb11782988af7c4c, REOPEN/MOVE 2024-11-12T18:14:50,737 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true 2024-11-12T18:14:50,738 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T18:14:50,738 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-12T18:14:50,738 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=65 2024-11-12T18:14:50,738 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=9911683f163c,34847,1731435282512, table=hbase:meta, region=1588230740 2024-11-12T18:14:50,738 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9911683f163c,34847,1731435282512, state=OPEN 2024-11-12T18:14:50,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,740 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:14:50,740 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,740 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:14:50,741 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731435320741 2024-11-12T18:14:50,741 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-12T18:14:50,742 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:50,743 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T18:14:50,745 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=60, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=70, state=RUNNABLE, hasLock=true; OpenRegionProcedure 4b703e982cd6bbf72ec2f771b120c649, server=9911683f163c,45243,1731435282573:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 9911683f163c,40543,1731435282322, splitWal=true, meta=true exec-time=2.3150 sec 2024-11-12T18:14:50,746 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5dcfe82d27e00f564e4ad71ea2e78f5c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,746 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2248a0d1296d1dbe4f8c9805798db03c, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,746 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a62a194de5221e9f84b2cfd2e83a46a, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,746 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e290ad15dec09a0cb11782988af7c4c, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=5 2024-11-12T18:14:50,746 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0b5ce997df05cef2b0f3fd25e49fafc, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,746 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4aff96e33c9923b5d2448f72171fd310, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,746 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=26fc65f8daf71d5831131b1bc6746a63, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,747 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dff0a9c660bd5084b34753ea40ea3278, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,747 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96e9ca1050c6b52ea6556fdb4b5d5f39, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,747 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a6d3052a2b28af2dab58e9247bb22b7, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,747 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d59e820eb0eef2601f5990d32a8adbc9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,747 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=394b0963ad0828345e14c3be23cb7790, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,747 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c739eeba765df343af4ef63444655c9f, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,747 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b703e982cd6bbf72ec2f771b120c649, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,747 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40b076d46dbfff2e1f3c89ad5c3f814d, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fbe24cda440bbf727cd581ab33ad7a9, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6d7c21378c2e7788ac010db26909da83, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1b81a9be48fbe50dde523592a87a4611, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4795f188f4931ea02df11d7474e4daa4, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e38ac98b32f350d84df37b6915c56a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7b0558136890d45b56827e5af943cfa, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=56e168ba1a3103978bee5b6c43feba36, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec76766d593d312811f043efb129a5a3, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0178ff26f4e2d0ab40d50fe6fcd026be, regionState=OPEN, lastHost=9911683f163c,45243,1731435282573, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fde2b7fbe9fa7c8a87ab544f5542d82a, regionState=OPENING, lastHost=9911683f163c,40543,1731435282322, regionLocation=9911683f163c,45243,1731435282573, openSeqNum=2 2024-11-12T18:14:50,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e4348dabb8addd0673ec5266827d8b4d, regionState=OPEN, lastHost=9911683f163c,34847,1731435282512, regionLocation=9911683f163c,34847,1731435282512, openSeqNum=2 2024-11-12T18:14:50,749 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-12T18:14:50,749 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731435350749 2024-11-12T18:14:50,749 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731435410749 2024-11-12T18:14:50,749 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-12T18:14:50,770 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=350 (was 375), OpenFileDescriptor=604 (was 618), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=759 (was 799), ProcessCount=11 (was 11), AvailableMemoryMB=2060 (was 2114) 2024-11-12T18:14:50,771 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-12T18:14:50,771 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T18:14:50,771 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T18:14:50,771 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:50,771 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:50,772 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:14:50,772 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-12T18:14:50,772 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=840105357, stopped=false 2024-11-12T18:14:50,772 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=9911683f163c,36779,1731435281367 2024-11-12T18:14:50,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T18:14:50,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T18:14:50,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T18:14:50,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T18:14:50,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T18:14:50,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T18:14:50,774 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:50,775 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T18:14:50,775 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T18:14:50,775 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:50,775 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T18:14:50,775 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9911683f163c,34847,1731435282512' ***** 2024-11-12T18:14:50,775 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-12T18:14:50,775 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9911683f163c,45243,1731435282573' ***** 2024-11-12T18:14:50,775 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-12T18:14:50,775 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T18:14:50,775 INFO [RS:2;9911683f163c:45243 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-12T18:14:50,775 INFO [RS:1;9911683f163c:34847 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-12T18:14:50,776 INFO [RS:2;9911683f163c:45243 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-12T18:14:50,776 INFO [RS:1;9911683f163c:34847 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-12T18:14:50,776 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-12T18:14:50,776 INFO [RS:1;9911683f163c:34847 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-12T18:14:50,776 INFO [RS:2;9911683f163c:45243 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-12T18:14:50,776 INFO [RS:2;9911683f163c:45243 {}] regionserver.HRegionServer(3091): Received CLOSE for 0178ff26f4e2d0ab40d50fe6fcd026be 2024-11-12T18:14:50,776 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-12T18:14:50,776 INFO [RS:1;9911683f163c:34847 {}] regionserver.HRegionServer(3091): Received CLOSE for e4348dabb8addd0673ec5266827d8b4d 2024-11-12T18:14:50,777 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T18:14:50,777 INFO [RS:2;9911683f163c:45243 {}] regionserver.HRegionServer(3091): Received CLOSE for 5dcfe82d27e00f564e4ad71ea2e78f5c 2024-11-12T18:14:50,777 INFO [RS:1;9911683f163c:34847 {}] regionserver.HRegionServer(3091): Received CLOSE for 96e9ca1050c6b52ea6556fdb4b5d5f39 2024-11-12T18:14:50,777 INFO [RS:2;9911683f163c:45243 {}] regionserver.HRegionServer(3091): Received CLOSE for 4795f188f4931ea02df11d7474e4daa4 2024-11-12T18:14:50,777 INFO [RS:1;9911683f163c:34847 {}] regionserver.HRegionServer(3091): Received CLOSE for 6d7c21378c2e7788ac010db26909da83 2024-11-12T18:14:50,777 INFO [RS:2;9911683f163c:45243 {}] regionserver.HRegionServer(3091): Received CLOSE for 40b076d46dbfff2e1f3c89ad5c3f814d 2024-11-12T18:14:50,777 INFO [RS:1;9911683f163c:34847 {}] regionserver.HRegionServer(3091): Received CLOSE for 6a6d3052a2b28af2dab58e9247bb22b7 2024-11-12T18:14:50,777 INFO [RS:2;9911683f163c:45243 {}] regionserver.HRegionServer(3091): Received CLOSE for 394b0963ad0828345e14c3be23cb7790 2024-11-12T18:14:50,777 INFO [RS:1;9911683f163c:34847 {}] regionserver.HRegionServer(3091): Received CLOSE for d59e820eb0eef2601f5990d32a8adbc9 2024-11-12T18:14:50,777 INFO [RS:2;9911683f163c:45243 {}] regionserver.HRegionServer(3091): Received CLOSE for fde2b7fbe9fa7c8a87ab544f5542d82a 2024-11-12T18:14:50,777 INFO [RS:1;9911683f163c:34847 {}] regionserver.HRegionServer(3091): Received CLOSE for ec76766d593d312811f043efb129a5a3 2024-11-12T18:14:50,777 INFO [RS:2;9911683f163c:45243 {}] regionserver.HRegionServer(3091): Received CLOSE for 4b703e982cd6bbf72ec2f771b120c649 2024-11-12T18:14:50,777 INFO [RS:1;9911683f163c:34847 {}] regionserver.HRegionServer(3091): Received CLOSE for 3a62a194de5221e9f84b2cfd2e83a46a 2024-11-12T18:14:50,777 INFO [RS:2;9911683f163c:45243 {}] regionserver.HRegionServer(3091): Received CLOSE for 6e290ad15dec09a0cb11782988af7c4c 2024-11-12T18:14:50,777 INFO [RS:1;9911683f163c:34847 {}] regionserver.HRegionServer(3091): Received CLOSE for c739eeba765df343af4ef63444655c9f 2024-11-12T18:14:50,777 INFO [RS:2;9911683f163c:45243 {}] regionserver.HRegionServer(3091): Received CLOSE for dff0a9c660bd5084b34753ea40ea3278 2024-11-12T18:14:50,777 INFO [RS:1;9911683f163c:34847 {}] regionserver.HRegionServer(3091): Received CLOSE for 4aff96e33c9923b5d2448f72171fd310 2024-11-12T18:14:50,777 INFO [RS:2;9911683f163c:45243 {}] regionserver.HRegionServer(3091): Received CLOSE for c0b5ce997df05cef2b0f3fd25e49fafc 2024-11-12T18:14:50,777 INFO [RS:1;9911683f163c:34847 {}] regionserver.HRegionServer(3091): Received CLOSE for 4e38ac98b32f350d84df37b6915c56a3 2024-11-12T18:14:50,777 INFO [RS:2;9911683f163c:45243 {}] regionserver.HRegionServer(3091): Received CLOSE for a7b0558136890d45b56827e5af943cfa 2024-11-12T18:14:50,777 INFO [RS:2;9911683f163c:45243 {}] regionserver.HRegionServer(959): stopping server 9911683f163c,45243,1731435282573 2024-11-12T18:14:50,777 INFO [RS:1;9911683f163c:34847 {}] regionserver.HRegionServer(959): stopping server 9911683f163c,34847,1731435282512 2024-11-12T18:14:50,777 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0178ff26f4e2d0ab40d50fe6fcd026be, disabling compactions & flushes 2024-11-12T18:14:50,777 INFO [RS:1;9911683f163c:34847 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T18:14:50,777 INFO [RS:2;9911683f163c:45243 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T18:14:50,777 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1731435284929.0178ff26f4e2d0ab40d50fe6fcd026be. 2024-11-12T18:14:50,777 INFO [RS:1;9911683f163c:34847 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;9911683f163c:34847. 2024-11-12T18:14:50,777 INFO [RS:2;9911683f163c:45243 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;9911683f163c:45243. 2024-11-12T18:14:50,778 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1731435284929.0178ff26f4e2d0ab40d50fe6fcd026be. 2024-11-12T18:14:50,778 DEBUG [RS:1;9911683f163c:34847 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T18:14:50,778 DEBUG [RS:2;9911683f163c:45243 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T18:14:50,778 DEBUG [RS:1;9911683f163c:34847 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:50,778 DEBUG [RS:2;9911683f163c:45243 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:50,778 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1731435284929.0178ff26f4e2d0ab40d50fe6fcd026be. after waiting 0 ms 2024-11-12T18:14:50,778 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1731435284929.0178ff26f4e2d0ab40d50fe6fcd026be. 2024-11-12T18:14:50,778 INFO [RS:1;9911683f163c:34847 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-12T18:14:50,778 INFO [RS:2;9911683f163c:45243 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-11-12T18:14:50,778 INFO [RS:1;9911683f163c:34847 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-12T18:14:50,778 INFO [RS:1;9911683f163c:34847 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-12T18:14:50,778 INFO [RS:1;9911683f163c:34847 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-12T18:14:50,778 DEBUG [RS:2;9911683f163c:45243 {}] regionserver.HRegionServer(1351): Waiting on 0178ff26f4e2d0ab40d50fe6fcd026be, 26fc65f8daf71d5831131b1bc6746a63, 394b0963ad0828345e14c3be23cb7790, 40b076d46dbfff2e1f3c89ad5c3f814d, 4795f188f4931ea02df11d7474e4daa4, 4b703e982cd6bbf72ec2f771b120c649, 5dcfe82d27e00f564e4ad71ea2e78f5c, 6e290ad15dec09a0cb11782988af7c4c, a7b0558136890d45b56827e5af943cfa, c0b5ce997df05cef2b0f3fd25e49fafc, dff0a9c660bd5084b34753ea40ea3278, fde2b7fbe9fa7c8a87ab544f5542d82a 2024-11-12T18:14:50,778 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e4348dabb8addd0673ec5266827d8b4d, disabling compactions & flushes 2024-11-12T18:14:50,779 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1731435284929.e4348dabb8addd0673ec5266827d8b4d. 2024-11-12T18:14:50,779 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1731435284929.e4348dabb8addd0673ec5266827d8b4d. 2024-11-12T18:14:50,779 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1731435284929.e4348dabb8addd0673ec5266827d8b4d. after waiting 0 ms 2024-11-12T18:14:50,779 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1731435284929.e4348dabb8addd0673ec5266827d8b4d. 2024-11-12T18:14:50,779 INFO [RS:1;9911683f163c:34847 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-11-12T18:14:50,780 DEBUG [RS:1;9911683f163c:34847 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 3a62a194de5221e9f84b2cfd2e83a46a, 4aff96e33c9923b5d2448f72171fd310, 4e38ac98b32f350d84df37b6915c56a3, 6a6d3052a2b28af2dab58e9247bb22b7, 6d7c21378c2e7788ac010db26909da83, 96e9ca1050c6b52ea6556fdb4b5d5f39, 9fbe24cda440bbf727cd581ab33ad7a9, c739eeba765df343af4ef63444655c9f, d59e820eb0eef2601f5990d32a8adbc9, e4348dabb8addd0673ec5266827d8b4d, ec76766d593d312811f043efb129a5a3 2024-11-12T18:14:50,780 DEBUG [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T18:14:50,780 INFO [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T18:14:50,780 DEBUG [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T18:14:50,780 DEBUG [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T18:14:50,780 DEBUG [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T18:14:50,780 INFO [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.66 KB heapSize=5.44 KB 2024-11-12T18:14:50,785 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/e4348dabb8addd0673ec5266827d8b4d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T18:14:50,786 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1731435284929.e4348dabb8addd0673ec5266827d8b4d. 2024-11-12T18:14:50,786 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e4348dabb8addd0673ec5266827d8b4d: Waiting for close lock at 1731435290778Running coprocessor pre-close hooks at 1731435290778Disabling compacts and flushes for region at 1731435290778Disabling writes for close at 1731435290779 (+1 ms)Writing region close event to WAL at 1731435290780 (+1 ms)Running coprocessor post-close hooks at 1731435290786 (+6 ms)Closed at 1731435290786 2024-11-12T18:14:50,786 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1731435284929.e4348dabb8addd0673ec5266827d8b4d. 2024-11-12T18:14:50,786 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 96e9ca1050c6b52ea6556fdb4b5d5f39, disabling compactions & flushes 2024-11-12T18:14:50,787 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1731435284929.96e9ca1050c6b52ea6556fdb4b5d5f39. 2024-11-12T18:14:50,787 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1731435284929.96e9ca1050c6b52ea6556fdb4b5d5f39. 2024-11-12T18:14:50,787 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1731435284929.96e9ca1050c6b52ea6556fdb4b5d5f39. after waiting 0 ms 2024-11-12T18:14:50,787 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1731435284929.96e9ca1050c6b52ea6556fdb4b5d5f39. 2024-11-12T18:14:50,787 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/0178ff26f4e2d0ab40d50fe6fcd026be/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T18:14:50,787 INFO [regionserver/9911683f163c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T18:14:50,788 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1731435284929.0178ff26f4e2d0ab40d50fe6fcd026be. 2024-11-12T18:14:50,788 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0178ff26f4e2d0ab40d50fe6fcd026be: Waiting for close lock at 1731435290777Running coprocessor pre-close hooks at 1731435290777Disabling compacts and flushes for region at 1731435290777Disabling writes for close at 1731435290778 (+1 ms)Writing region close event to WAL at 1731435290779 (+1 ms)Running coprocessor post-close hooks at 1731435290787 (+8 ms)Closed at 1731435290788 (+1 ms) 2024-11-12T18:14:50,788 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1731435284929.0178ff26f4e2d0ab40d50fe6fcd026be. 2024-11-12T18:14:50,788 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5dcfe82d27e00f564e4ad71ea2e78f5c, disabling compactions & flushes 2024-11-12T18:14:50,788 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1731435284929.5dcfe82d27e00f564e4ad71ea2e78f5c. 2024-11-12T18:14:50,788 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1731435284929.5dcfe82d27e00f564e4ad71ea2e78f5c. 2024-11-12T18:14:50,788 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1731435284929.5dcfe82d27e00f564e4ad71ea2e78f5c. after waiting 0 ms 2024-11-12T18:14:50,788 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1731435284929.5dcfe82d27e00f564e4ad71ea2e78f5c. 2024-11-12T18:14:50,790 INFO [regionserver/9911683f163c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T18:14:50,793 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/96e9ca1050c6b52ea6556fdb4b5d5f39/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T18:14:50,794 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1731435284929.96e9ca1050c6b52ea6556fdb4b5d5f39. 2024-11-12T18:14:50,794 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 96e9ca1050c6b52ea6556fdb4b5d5f39: Waiting for close lock at 1731435290786Running coprocessor pre-close hooks at 1731435290786Disabling compacts and flushes for region at 1731435290786Disabling writes for close at 1731435290787 (+1 ms)Writing region close event to WAL at 1731435290787Running coprocessor post-close hooks at 1731435290794 (+7 ms)Closed at 1731435290794 2024-11-12T18:14:50,794 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1731435284929.96e9ca1050c6b52ea6556fdb4b5d5f39. 2024-11-12T18:14:50,794 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6d7c21378c2e7788ac010db26909da83, disabling compactions & flushes 2024-11-12T18:14:50,794 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1731435284929.6d7c21378c2e7788ac010db26909da83. 2024-11-12T18:14:50,794 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1731435284929.6d7c21378c2e7788ac010db26909da83. 2024-11-12T18:14:50,794 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1731435284929.6d7c21378c2e7788ac010db26909da83. after waiting 0 ms 2024-11-12T18:14:50,794 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1731435284929.6d7c21378c2e7788ac010db26909da83. 2024-11-12T18:14:50,795 DEBUG [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/hbase/meta/1588230740/.tmp/info/2f877e1c6f094dbf9734143159595d7a is 138, key is test,bbb,1731435284929.2248a0d1296d1dbe4f8c9805798db03c./info:regioninfo/1731435289781/Put/seqid=0 2024-11-12T18:14:50,801 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/5dcfe82d27e00f564e4ad71ea2e78f5c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T18:14:50,802 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1731435284929.5dcfe82d27e00f564e4ad71ea2e78f5c. 2024-11-12T18:14:50,802 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5dcfe82d27e00f564e4ad71ea2e78f5c: Waiting for close lock at 1731435290788Running coprocessor pre-close hooks at 1731435290788Disabling compacts and flushes for region at 1731435290788Disabling writes for close at 1731435290788Writing region close event to WAL at 1731435290789 (+1 ms)Running coprocessor post-close hooks at 1731435290802 (+13 ms)Closed at 1731435290802 2024-11-12T18:14:50,802 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1731435284929.5dcfe82d27e00f564e4ad71ea2e78f5c. 2024-11-12T18:14:50,802 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4795f188f4931ea02df11d7474e4daa4, disabling compactions & flushes 2024-11-12T18:14:50,803 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1731435284929.4795f188f4931ea02df11d7474e4daa4. 2024-11-12T18:14:50,803 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1731435284929.4795f188f4931ea02df11d7474e4daa4. 2024-11-12T18:14:50,803 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1731435284929.4795f188f4931ea02df11d7474e4daa4. after waiting 0 ms 2024-11-12T18:14:50,803 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1731435284929.4795f188f4931ea02df11d7474e4daa4. 2024-11-12T18:14:50,822 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/6d7c21378c2e7788ac010db26909da83/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T18:14:50,823 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1731435284929.6d7c21378c2e7788ac010db26909da83. 2024-11-12T18:14:50,824 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6d7c21378c2e7788ac010db26909da83: Waiting for close lock at 1731435290794Running coprocessor pre-close hooks at 1731435290794Disabling compacts and flushes for region at 1731435290794Disabling writes for close at 1731435290794Writing region close event to WAL at 1731435290800 (+6 ms)Running coprocessor post-close hooks at 1731435290823 (+23 ms)Closed at 1731435290823 2024-11-12T18:14:50,824 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1731435284929.6d7c21378c2e7788ac010db26909da83. 2024-11-12T18:14:50,824 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6a6d3052a2b28af2dab58e9247bb22b7, disabling compactions & flushes 2024-11-12T18:14:50,824 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1731435284929.6a6d3052a2b28af2dab58e9247bb22b7. 2024-11-12T18:14:50,824 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1731435284929.6a6d3052a2b28af2dab58e9247bb22b7. 2024-11-12T18:14:50,824 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1731435284929.6a6d3052a2b28af2dab58e9247bb22b7. after waiting 0 ms 2024-11-12T18:14:50,824 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1731435284929.6a6d3052a2b28af2dab58e9247bb22b7. 2024-11-12T18:14:50,829 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/4795f188f4931ea02df11d7474e4daa4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T18:14:50,830 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1731435284929.4795f188f4931ea02df11d7474e4daa4. 2024-11-12T18:14:50,830 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4795f188f4931ea02df11d7474e4daa4: Waiting for close lock at 1731435290802Running coprocessor pre-close hooks at 1731435290802Disabling compacts and flushes for region at 1731435290802Disabling writes for close at 1731435290803 (+1 ms)Writing region close event to WAL at 1731435290821 (+18 ms)Running coprocessor post-close hooks at 1731435290830 (+9 ms)Closed at 1731435290830 2024-11-12T18:14:50,830 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1731435284929.4795f188f4931ea02df11d7474e4daa4. 2024-11-12T18:14:50,831 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 40b076d46dbfff2e1f3c89ad5c3f814d, disabling compactions & flushes 2024-11-12T18:14:50,831 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1731435284929.40b076d46dbfff2e1f3c89ad5c3f814d. 2024-11-12T18:14:50,831 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1731435284929.40b076d46dbfff2e1f3c89ad5c3f814d. 2024-11-12T18:14:50,831 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1731435284929.40b076d46dbfff2e1f3c89ad5c3f814d. after waiting 0 ms 2024-11-12T18:14:50,831 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1731435284929.40b076d46dbfff2e1f3c89ad5c3f814d. 2024-11-12T18:14:50,842 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/6a6d3052a2b28af2dab58e9247bb22b7/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T18:14:50,843 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1731435284929.6a6d3052a2b28af2dab58e9247bb22b7. 2024-11-12T18:14:50,843 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6a6d3052a2b28af2dab58e9247bb22b7: Waiting for close lock at 1731435290824Running coprocessor pre-close hooks at 1731435290824Disabling compacts and flushes for region at 1731435290824Disabling writes for close at 1731435290824Writing region close event to WAL at 1731435290825 (+1 ms)Running coprocessor post-close hooks at 1731435290843 (+18 ms)Closed at 1731435290843 2024-11-12T18:14:50,843 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1731435284929.6a6d3052a2b28af2dab58e9247bb22b7. 2024-11-12T18:14:50,843 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d59e820eb0eef2601f5990d32a8adbc9, disabling compactions & flushes 2024-11-12T18:14:50,843 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1731435284929.d59e820eb0eef2601f5990d32a8adbc9. 2024-11-12T18:14:50,843 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1731435284929.d59e820eb0eef2601f5990d32a8adbc9. 2024-11-12T18:14:50,843 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1731435284929.d59e820eb0eef2601f5990d32a8adbc9. after waiting 0 ms 2024-11-12T18:14:50,843 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1731435284929.d59e820eb0eef2601f5990d32a8adbc9. 2024-11-12T18:14:50,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741869_1045 (size=8195) 2024-11-12T18:14:50,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741869_1045 (size=8195) 2024-11-12T18:14:50,851 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/40b076d46dbfff2e1f3c89ad5c3f814d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T18:14:50,852 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1731435284929.40b076d46dbfff2e1f3c89ad5c3f814d. 2024-11-12T18:14:50,852 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 40b076d46dbfff2e1f3c89ad5c3f814d: Waiting for close lock at 1731435290831Running coprocessor pre-close hooks at 1731435290831Disabling compacts and flushes for region at 1731435290831Disabling writes for close at 1731435290831Writing region close event to WAL at 1731435290841 (+10 ms)Running coprocessor post-close hooks at 1731435290852 (+11 ms)Closed at 1731435290852 2024-11-12T18:14:50,852 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1731435284929.40b076d46dbfff2e1f3c89ad5c3f814d. 2024-11-12T18:14:50,852 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 394b0963ad0828345e14c3be23cb7790, disabling compactions & flushes 2024-11-12T18:14:50,852 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1731435284929.394b0963ad0828345e14c3be23cb7790. 2024-11-12T18:14:50,852 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1731435284929.394b0963ad0828345e14c3be23cb7790. 2024-11-12T18:14:50,852 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1731435284929.394b0963ad0828345e14c3be23cb7790. after waiting 0 ms 2024-11-12T18:14:50,852 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1731435284929.394b0963ad0828345e14c3be23cb7790. 2024-11-12T18:14:50,861 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/d59e820eb0eef2601f5990d32a8adbc9/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T18:14:50,862 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1731435284929.d59e820eb0eef2601f5990d32a8adbc9. 2024-11-12T18:14:50,862 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d59e820eb0eef2601f5990d32a8adbc9: Waiting for close lock at 1731435290843Running coprocessor pre-close hooks at 1731435290843Disabling compacts and flushes for region at 1731435290843Disabling writes for close at 1731435290843Writing region close event to WAL at 1731435290849 (+6 ms)Running coprocessor post-close hooks at 1731435290862 (+13 ms)Closed at 1731435290862 2024-11-12T18:14:50,862 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/394b0963ad0828345e14c3be23cb7790/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T18:14:50,863 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1731435284929.d59e820eb0eef2601f5990d32a8adbc9. 2024-11-12T18:14:50,863 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ec76766d593d312811f043efb129a5a3, disabling compactions & flushes 2024-11-12T18:14:50,863 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1731435284929.ec76766d593d312811f043efb129a5a3. 2024-11-12T18:14:50,863 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1731435284929.ec76766d593d312811f043efb129a5a3. 2024-11-12T18:14:50,863 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1731435284929.ec76766d593d312811f043efb129a5a3. after waiting 0 ms 2024-11-12T18:14:50,863 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1731435284929.ec76766d593d312811f043efb129a5a3. 2024-11-12T18:14:50,864 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1731435284929.394b0963ad0828345e14c3be23cb7790. 2024-11-12T18:14:50,864 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 394b0963ad0828345e14c3be23cb7790: Waiting for close lock at 1731435290852Running coprocessor pre-close hooks at 1731435290852Disabling compacts and flushes for region at 1731435290852Disabling writes for close at 1731435290852Writing region close event to WAL at 1731435290854 (+2 ms)Running coprocessor post-close hooks at 1731435290864 (+10 ms)Closed at 1731435290864 2024-11-12T18:14:50,864 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1731435284929.394b0963ad0828345e14c3be23cb7790. 2024-11-12T18:14:50,864 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fde2b7fbe9fa7c8a87ab544f5542d82a, disabling compactions & flushes 2024-11-12T18:14:50,864 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1731435284929.fde2b7fbe9fa7c8a87ab544f5542d82a. 2024-11-12T18:14:50,864 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1731435284929.fde2b7fbe9fa7c8a87ab544f5542d82a. 2024-11-12T18:14:50,864 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1731435284929.fde2b7fbe9fa7c8a87ab544f5542d82a. after waiting 0 ms 2024-11-12T18:14:50,865 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1731435284929.fde2b7fbe9fa7c8a87ab544f5542d82a. 2024-11-12T18:14:50,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46767 is added to blk_1073741869_1045 (size=8195) 2024-11-12T18:14:50,871 INFO [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.66 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/hbase/meta/1588230740/.tmp/info/2f877e1c6f094dbf9734143159595d7a 2024-11-12T18:14:50,877 INFO [regionserver/9911683f163c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-12T18:14:50,878 INFO [regionserver/9911683f163c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-12T18:14:50,881 INFO [regionserver/9911683f163c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-12T18:14:50,881 INFO [regionserver/9911683f163c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-12T18:14:50,885 DEBUG [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/hbase/meta/1588230740/.tmp/info/2f877e1c6f094dbf9734143159595d7a as hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/hbase/meta/1588230740/info/2f877e1c6f094dbf9734143159595d7a 2024-11-12T18:14:50,889 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/fde2b7fbe9fa7c8a87ab544f5542d82a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T18:14:50,891 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1731435284929.fde2b7fbe9fa7c8a87ab544f5542d82a. 2024-11-12T18:14:50,891 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fde2b7fbe9fa7c8a87ab544f5542d82a: Waiting for close lock at 1731435290864Running coprocessor pre-close hooks at 1731435290864Disabling compacts and flushes for region at 1731435290864Disabling writes for close at 1731435290864Writing region close event to WAL at 1731435290878 (+14 ms)Running coprocessor post-close hooks at 1731435290890 (+12 ms)Closed at 1731435290891 (+1 ms) 2024-11-12T18:14:50,891 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1731435284929.fde2b7fbe9fa7c8a87ab544f5542d82a. 2024-11-12T18:14:50,891 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/ec76766d593d312811f043efb129a5a3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T18:14:50,891 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4b703e982cd6bbf72ec2f771b120c649, disabling compactions & flushes 2024-11-12T18:14:50,892 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1731435284929.4b703e982cd6bbf72ec2f771b120c649. 2024-11-12T18:14:50,892 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1731435284929.4b703e982cd6bbf72ec2f771b120c649. 2024-11-12T18:14:50,892 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1731435284929.4b703e982cd6bbf72ec2f771b120c649. after waiting 0 ms 2024-11-12T18:14:50,893 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1731435284929.4b703e982cd6bbf72ec2f771b120c649. 2024-11-12T18:14:50,893 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1731435284929.ec76766d593d312811f043efb129a5a3. 2024-11-12T18:14:50,893 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ec76766d593d312811f043efb129a5a3: Waiting for close lock at 1731435290863Running coprocessor pre-close hooks at 1731435290863Disabling compacts and flushes for region at 1731435290863Disabling writes for close at 1731435290863Writing region close event to WAL at 1731435290870 (+7 ms)Running coprocessor post-close hooks at 1731435290893 (+23 ms)Closed at 1731435290893 2024-11-12T18:14:50,893 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1731435284929.ec76766d593d312811f043efb129a5a3. 2024-11-12T18:14:50,893 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3a62a194de5221e9f84b2cfd2e83a46a, disabling compactions & flushes 2024-11-12T18:14:50,893 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1731435284929.3a62a194de5221e9f84b2cfd2e83a46a. 2024-11-12T18:14:50,893 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1731435284929.3a62a194de5221e9f84b2cfd2e83a46a. 2024-11-12T18:14:50,893 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1731435284929.3a62a194de5221e9f84b2cfd2e83a46a. after waiting 0 ms 2024-11-12T18:14:50,893 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1731435284929.3a62a194de5221e9f84b2cfd2e83a46a. 2024-11-12T18:14:50,897 INFO [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/hbase/meta/1588230740/info/2f877e1c6f094dbf9734143159595d7a, entries=24, sequenceid=75, filesize=8.0 K 2024-11-12T18:14:50,898 INFO [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 118ms, sequenceid=75, compaction requested=false 2024-11-12T18:14:50,900 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/3a62a194de5221e9f84b2cfd2e83a46a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T18:14:50,900 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1731435284929.3a62a194de5221e9f84b2cfd2e83a46a. 2024-11-12T18:14:50,900 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3a62a194de5221e9f84b2cfd2e83a46a: Waiting for close lock at 1731435290893Running coprocessor pre-close hooks at 1731435290893Disabling compacts and flushes for region at 1731435290893Disabling writes for close at 1731435290893Writing region close event to WAL at 1731435290894 (+1 ms)Running coprocessor post-close hooks at 1731435290900 (+6 ms)Closed at 1731435290900 2024-11-12T18:14:50,901 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1731435284929.3a62a194de5221e9f84b2cfd2e83a46a. 2024-11-12T18:14:50,901 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c739eeba765df343af4ef63444655c9f, disabling compactions & flushes 2024-11-12T18:14:50,901 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1731435284929.c739eeba765df343af4ef63444655c9f. 2024-11-12T18:14:50,901 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1731435284929.c739eeba765df343af4ef63444655c9f. 2024-11-12T18:14:50,901 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1731435284929.c739eeba765df343af4ef63444655c9f. after waiting 0 ms 2024-11-12T18:14:50,901 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1731435284929.c739eeba765df343af4ef63444655c9f. 2024-11-12T18:14:50,901 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/4b703e982cd6bbf72ec2f771b120c649/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T18:14:50,902 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1731435284929.4b703e982cd6bbf72ec2f771b120c649. 2024-11-12T18:14:50,902 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4b703e982cd6bbf72ec2f771b120c649: Waiting for close lock at 1731435290891Running coprocessor pre-close hooks at 1731435290891Disabling compacts and flushes for region at 1731435290891Disabling writes for close at 1731435290893 (+2 ms)Writing region close event to WAL at 1731435290894 (+1 ms)Running coprocessor post-close hooks at 1731435290902 (+8 ms)Closed at 1731435290902 2024-11-12T18:14:50,902 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1731435284929.4b703e982cd6bbf72ec2f771b120c649. 2024-11-12T18:14:50,902 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6e290ad15dec09a0cb11782988af7c4c, disabling compactions & flushes 2024-11-12T18:14:50,902 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1731435284929.6e290ad15dec09a0cb11782988af7c4c. 2024-11-12T18:14:50,902 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1731435284929.6e290ad15dec09a0cb11782988af7c4c. 2024-11-12T18:14:50,903 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1731435284929.6e290ad15dec09a0cb11782988af7c4c. after waiting 0 ms 2024-11-12T18:14:50,903 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1731435284929.6e290ad15dec09a0cb11782988af7c4c. 2024-11-12T18:14:50,917 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/c739eeba765df343af4ef63444655c9f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T18:14:50,918 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1731435284929.c739eeba765df343af4ef63444655c9f. 2024-11-12T18:14:50,919 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/6e290ad15dec09a0cb11782988af7c4c/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-12T18:14:50,919 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c739eeba765df343af4ef63444655c9f: Waiting for close lock at 1731435290901Running coprocessor pre-close hooks at 1731435290901Disabling compacts and flushes for region at 1731435290901Disabling writes for close at 1731435290901Writing region close event to WAL at 1731435290906 (+5 ms)Running coprocessor post-close hooks at 1731435290918 (+12 ms)Closed at 1731435290918 2024-11-12T18:14:50,919 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1731435284929.c739eeba765df343af4ef63444655c9f. 2024-11-12T18:14:50,919 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4aff96e33c9923b5d2448f72171fd310, disabling compactions & flushes 2024-11-12T18:14:50,919 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1731435284929.4aff96e33c9923b5d2448f72171fd310. 2024-11-12T18:14:50,919 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1731435284929.4aff96e33c9923b5d2448f72171fd310. 2024-11-12T18:14:50,919 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1731435284929.4aff96e33c9923b5d2448f72171fd310. after waiting 0 ms 2024-11-12T18:14:50,919 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1731435284929.4aff96e33c9923b5d2448f72171fd310. 2024-11-12T18:14:50,920 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1731435284929.6e290ad15dec09a0cb11782988af7c4c. 2024-11-12T18:14:50,920 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6e290ad15dec09a0cb11782988af7c4c: Waiting for close lock at 1731435290902Running coprocessor pre-close hooks at 1731435290902Disabling compacts and flushes for region at 1731435290902Disabling writes for close at 1731435290903 (+1 ms)Writing region close event to WAL at 1731435290911 (+8 ms)Running coprocessor post-close hooks at 1731435290920 (+9 ms)Closed at 1731435290920 2024-11-12T18:14:50,920 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1731435284929.6e290ad15dec09a0cb11782988af7c4c. 2024-11-12T18:14:50,920 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing dff0a9c660bd5084b34753ea40ea3278, disabling compactions & flushes 2024-11-12T18:14:50,920 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1731435284929.dff0a9c660bd5084b34753ea40ea3278. 2024-11-12T18:14:50,920 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1731435284929.dff0a9c660bd5084b34753ea40ea3278. 2024-11-12T18:14:50,920 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1731435284929.dff0a9c660bd5084b34753ea40ea3278. after waiting 0 ms 2024-11-12T18:14:50,920 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1731435284929.dff0a9c660bd5084b34753ea40ea3278. 2024-11-12T18:14:50,926 DEBUG [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/hbase/meta/1588230740/recovered.edits/78.seqid, newMaxSeqId=78, maxSeqId=64 2024-11-12T18:14:50,927 DEBUG [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T18:14:50,927 INFO [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T18:14:50,927 DEBUG [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731435290780Running coprocessor pre-close hooks at 1731435290780Disabling compacts and flushes for region at 1731435290780Disabling writes for close at 1731435290780Obtaining lock to block concurrent updates at 1731435290780Preparing flush snapshotting stores in 1588230740 at 1731435290780Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2720, getHeapSize=5504, getOffHeapSize=0, getCellsCount=24 at 1731435290786 (+6 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731435290789 (+3 ms)Flushing 1588230740/info: creating writer at 1731435290789Flushing 1588230740/info: appending metadata at 1731435290794 (+5 ms)Flushing 1588230740/info: closing flushed file at 1731435290794Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e7537be: reopening flushed file at 1731435290881 (+87 ms)Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 118ms, sequenceid=75, compaction requested=false at 1731435290899 (+18 ms)Writing region close event to WAL at 1731435290917 (+18 ms)Running coprocessor post-close hooks at 1731435290926 (+9 ms)Closed at 1731435290927 (+1 ms) 2024-11-12T18:14:50,927 DEBUG [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-12T18:14:50,929 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/4aff96e33c9923b5d2448f72171fd310/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T18:14:50,930 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/dff0a9c660bd5084b34753ea40ea3278/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T18:14:50,930 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1731435284929.4aff96e33c9923b5d2448f72171fd310. 2024-11-12T18:14:50,930 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4aff96e33c9923b5d2448f72171fd310: Waiting for close lock at 1731435290919Running coprocessor pre-close hooks at 1731435290919Disabling compacts and flushes for region at 1731435290919Disabling writes for close at 1731435290919Writing region close event to WAL at 1731435290925 (+6 ms)Running coprocessor post-close hooks at 1731435290930 (+5 ms)Closed at 1731435290930 2024-11-12T18:14:50,930 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1731435284929.4aff96e33c9923b5d2448f72171fd310. 2024-11-12T18:14:50,930 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4e38ac98b32f350d84df37b6915c56a3, disabling compactions & flushes 2024-11-12T18:14:50,930 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1731435284929.4e38ac98b32f350d84df37b6915c56a3. 2024-11-12T18:14:50,931 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1731435284929.4e38ac98b32f350d84df37b6915c56a3. 2024-11-12T18:14:50,931 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1731435284929.4e38ac98b32f350d84df37b6915c56a3. after waiting 0 ms 2024-11-12T18:14:50,931 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1731435284929.4e38ac98b32f350d84df37b6915c56a3. 2024-11-12T18:14:50,931 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1731435284929.dff0a9c660bd5084b34753ea40ea3278. 2024-11-12T18:14:50,931 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for dff0a9c660bd5084b34753ea40ea3278: Waiting for close lock at 1731435290920Running coprocessor pre-close hooks at 1731435290920Disabling compacts and flushes for region at 1731435290920Disabling writes for close at 1731435290920Writing region close event to WAL at 1731435290925 (+5 ms)Running coprocessor post-close hooks at 1731435290931 (+6 ms)Closed at 1731435290931 2024-11-12T18:14:50,931 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1731435284929.dff0a9c660bd5084b34753ea40ea3278. 2024-11-12T18:14:50,931 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c0b5ce997df05cef2b0f3fd25e49fafc, disabling compactions & flushes 2024-11-12T18:14:50,931 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1731435284929.c0b5ce997df05cef2b0f3fd25e49fafc. 2024-11-12T18:14:50,931 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1731435284929.c0b5ce997df05cef2b0f3fd25e49fafc. 2024-11-12T18:14:50,931 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1731435284929.c0b5ce997df05cef2b0f3fd25e49fafc. after waiting 0 ms 2024-11-12T18:14:50,931 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1731435284929.c0b5ce997df05cef2b0f3fd25e49fafc. 2024-11-12T18:14:50,935 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/4e38ac98b32f350d84df37b6915c56a3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T18:14:50,935 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/c0b5ce997df05cef2b0f3fd25e49fafc/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T18:14:50,936 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1731435284929.4e38ac98b32f350d84df37b6915c56a3. 2024-11-12T18:14:50,936 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4e38ac98b32f350d84df37b6915c56a3: Waiting for close lock at 1731435290930Running coprocessor pre-close hooks at 1731435290930Disabling compacts and flushes for region at 1731435290930Disabling writes for close at 1731435290931 (+1 ms)Writing region close event to WAL at 1731435290931Running coprocessor post-close hooks at 1731435290936 (+5 ms)Closed at 1731435290936 2024-11-12T18:14:50,936 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1731435284929.4e38ac98b32f350d84df37b6915c56a3. 2024-11-12T18:14:50,936 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1731435284929.c0b5ce997df05cef2b0f3fd25e49fafc. 2024-11-12T18:14:50,936 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c0b5ce997df05cef2b0f3fd25e49fafc: Waiting for close lock at 1731435290931Running coprocessor pre-close hooks at 1731435290931Disabling compacts and flushes for region at 1731435290931Disabling writes for close at 1731435290931Writing region close event to WAL at 1731435290932 (+1 ms)Running coprocessor post-close hooks at 1731435290936 (+4 ms)Closed at 1731435290936 2024-11-12T18:14:50,936 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1731435284929.c0b5ce997df05cef2b0f3fd25e49fafc. 2024-11-12T18:14:50,936 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a7b0558136890d45b56827e5af943cfa, disabling compactions & flushes 2024-11-12T18:14:50,936 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1731435284929.a7b0558136890d45b56827e5af943cfa. 2024-11-12T18:14:50,936 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1731435284929.a7b0558136890d45b56827e5af943cfa. 2024-11-12T18:14:50,936 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1731435284929.a7b0558136890d45b56827e5af943cfa. after waiting 0 ms 2024-11-12T18:14:50,936 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1731435284929.a7b0558136890d45b56827e5af943cfa. 2024-11-12T18:14:50,940 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/data/default/test/a7b0558136890d45b56827e5af943cfa/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T18:14:50,941 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1731435284929.a7b0558136890d45b56827e5af943cfa. 2024-11-12T18:14:50,941 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a7b0558136890d45b56827e5af943cfa: Waiting for close lock at 1731435290936Running coprocessor pre-close hooks at 1731435290936Disabling compacts and flushes for region at 1731435290936Disabling writes for close at 1731435290936Writing region close event to WAL at 1731435290937 (+1 ms)Running coprocessor post-close hooks at 1731435290941 (+4 ms)Closed at 1731435290941 2024-11-12T18:14:50,941 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1731435284929.a7b0558136890d45b56827e5af943cfa. 2024-11-12T18:14:50,979 INFO [RS:2;9911683f163c:45243 {}] regionserver.HRegionServer(976): stopping server 9911683f163c,45243,1731435282573; all regions closed. 2024-11-12T18:14:50,980 INFO [RS:1;9911683f163c:34847 {}] regionserver.HRegionServer(976): stopping server 9911683f163c,34847,1731435282512; all regions closed. 2024-11-12T18:14:50,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46767 is added to blk_1073741833_1009 (size=6258) 2024-11-12T18:14:50,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741833_1009 (size=6258) 2024-11-12T18:14:50,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741833_1009 (size=6258) 2024-11-12T18:14:50,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46767 is added to blk_1073741865_1041 (size=4204) 2024-11-12T18:14:50,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741865_1041 (size=4204) 2024-11-12T18:14:50,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741865_1041 (size=4204) 2024-11-12T18:14:50,992 DEBUG [RS:1;9911683f163c:34847 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/oldWALs 2024-11-12T18:14:50,992 INFO [RS:1;9911683f163c:34847 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9911683f163c%2C34847%2C1731435282512.meta:.meta(num 1731435289051) 2024-11-12T18:14:50,992 DEBUG [RS:2;9911683f163c:45243 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/oldWALs 2024-11-12T18:14:50,992 INFO [RS:2;9911683f163c:45243 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9911683f163c%2C45243%2C1731435282573:(num 1731435284059) 2024-11-12T18:14:50,992 DEBUG [RS:2;9911683f163c:45243 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:50,992 INFO [RS:2;9911683f163c:45243 {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T18:14:50,992 INFO [RS:2;9911683f163c:45243 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T18:14:50,993 INFO [RS:2;9911683f163c:45243 {}] hbase.ChoreService(370): Chore service for: regionserver/9911683f163c:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-12T18:14:50,993 INFO [RS:2;9911683f163c:45243 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-12T18:14:50,993 INFO [RS:2;9911683f163c:45243 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-12T18:14:50,993 INFO [RS:2;9911683f163c:45243 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-12T18:14:50,993 INFO [RS:2;9911683f163c:45243 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T18:14:50,994 INFO [regionserver/9911683f163c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T18:14:50,994 WARN [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.Threads(127): sleep interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2374) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:50,995 WARN [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,ggg,1731435284929.26fc65f8daf71d5831131b1bc6746a63., aborting... java.io.IOException: Failed to report opened region to master: test,ggg,1731435284929.26fc65f8daf71d5831131b1bc6746a63. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:50,995 ERROR [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2440): ***** ABORTING region server 9911683f163c,45243,1731435282573: Failed to open region test,ggg,1731435284929.26fc65f8daf71d5831131b1bc6746a63. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,ggg,1731435284929.26fc65f8daf71d5831131b1bc6746a63. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:50,995 ERROR [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-12T18:14:50,995 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/WALs/9911683f163c,34847,1731435282512/9911683f163c%2C34847%2C1731435282512.1731435284060 not finished, retry = 0 2024-11-12T18:14:50,995 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-12T18:14:50,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741835_1011 (size=5731) 2024-11-12T18:14:50,996 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-12T18:14:50,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741835_1011 (size=5731) 2024-11-12T18:14:50,998 INFO [RS:2;9911683f163c:45243 {}] executor.ExecutorService(108): Executor-9-RS_OPEN_REGION-regionserver/9911683f163c:0 had [Event #36 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler)] on shutdown 2024-11-12T18:14:50,998 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-12T18:14:50,998 INFO [RS:2;9911683f163c:45243 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:45243 2024-11-12T18:14:50,999 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-12T18:14:51,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9911683f163c,45243,1731435282573 2024-11-12T18:14:51,001 INFO [RS:2;9911683f163c:45243 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T18:14:51,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T18:14:51,001 ERROR [Time-limited test-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher. java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$354/0x00007feca88c1d18@6f1c267 rejected from java.util.concurrent.ThreadPoolExecutor@43f34cdd[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 39] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4] 2024-11-12T18:14:51,003 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9911683f163c,45243,1731435282573] 2024-11-12T18:14:51,005 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9911683f163c,45243,1731435282573 already deleted, retry=false 2024-11-12T18:14:51,005 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9911683f163c,45243,1731435282573 expired; onlineServers=1 2024-11-12T18:14:51,008 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 504917696 }, "NonHeapMemoryUsage": { "committed": 163643392, "init": 7667712, "max": -1, "used": 161030440 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "9911683f163c", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 39109, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 115, "ProcessCallTime_min": 0, "ProcessCallTime_max": 20, "ProcessCallTime_mean": 2, "ProcessCallTime_25th_percentile": 5, "ProcessCallTime_median": 10, "ProcessCallTime_75th_percentile": 15, "ProcessCallTime_90th_percentile": 18, "ProcessCallTime_95th_percentile": 19, "ProcessCallTime_98th_percentile": 19, "ProcessCallTime_99th_percentile": 19, "ProcessCallTime_99.9th_percentile": 19, "ProcessCallTime_TimeRangeCount_0-1": 115, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 115, "QueueCallTime_min": 0, "QueueCallTime_max": 12, "QueueCallTime_mean": 1, "QueueCallTime_25th_percentile": 3, "QueueCallTime_median": 6, "QueueCallTime_75th_percentile": 9, "QueueCallTime_90th_percentile": 10, "QueueCallTime_95th_percentile": 11, "QueueCallTime_98th_percentile": 11, "QueueCallTime_99th_percentile": 11, "QueueCallTime_99.9th_percentile": 11, "QueueCallTime_TimeRangeCount_0-1": 115, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 115, "TotalCallTime_min": 0, "TotalCallTime_max": 20, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 5, "TotalCallTime_median": 10, "TotalCallTime_75th_percentile": 15, "TotalCallTime_90th_percentile": 18, "TotalCallTime_95th_percentile": 19, "TotalCallTime_98th_percentile": 19, "TotalCallTime_99th_percentile": 19, "TotalCallTime_99.9th_percentile": 19, "TotalCallTime_TimeRangeCount_0-1": 115, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 115, "ResponseSize_min": 0, "ResponseSize_max": 22359, "ResponseSize_mean": 594, "ResponseSize_25th_percentile": 5589, "ResponseSize_median": 11179, "ResponseSize_75th_percentile": 16769, "ResponseSize_90th_percentile": 20123, "ResponseSize_95th_percentile": 21241, "ResponseSize_98th_percentile": 21911, "ResponseSize_99th_percentile": 22135, "ResponseSize_99.9th_percentile": 22336, "ResponseSize_SizeRangeCount_0-10": 115, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 115, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 308, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 115, "sentBytes": 54885 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "9911683f163c", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:49826", "tag.serverName": "9911683f163c,34847,1731435282512", "tag.clusterId": "8ba834a2-b172-4942-909b-d8381e92fea0", "tag.Context": "regionserver", "tag.Hostname": "9911683f163c", "regionCount": 11, "storeCount": 9, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 2304, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731435282512, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 3341, "localBytesRead": 3341, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.8, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 7474, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 17, "totalRowActionRequestCount": 9, "readRequestCount": 9, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 9, "rpcScanRequestCount": 9, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 8, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 9, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 9, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 8, "Put_min": 0, "Put_max": 3, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 1, "Put_75th_percentile": 2, "Put_90th_percentile": 2, "Put_95th_percentile": 2, "Put_98th_percentile": 2, "Put_99th_percentile": 2, "Put_99.9th_percentile": 2, "Put_TimeRangeCount_0-1": 8, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 46703, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 30111, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 46703, "FlushOutputSize_max": 46703, "FlushOutputSize_mean": 46703, "FlushOutputSize_25th_percentile": 46703, "FlushOutputSize_median": 46703, "FlushOutputSize_75th_percentile": 46703, "FlushOutputSize_90th_percentile": 46703, "FlushOutputSize_95th_percentile": 46703, "FlushOutputSize_98th_percentile": 46703, "FlushOutputSize_99th_percentile": 46703, "FlushOutputSize_99.9th_percentile": 46703, "FlushOutputSize_SizeRangeCount_0-10": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 30111, "FlushMemstoreSize_max": 30111, "FlushMemstoreSize_mean": 30111, "FlushMemstoreSize_25th_percentile": 30111, "FlushMemstoreSize_median": 30111, "FlushMemstoreSize_75th_percentile": 30111, "FlushMemstoreSize_90th_percentile": 30111, "FlushMemstoreSize_95th_percentile": 30111, "FlushMemstoreSize_98th_percentile": 30111, "FlushMemstoreSize_99th_percentile": 30111, "FlushMemstoreSize_99.9th_percentile": 30111, "FlushMemstoreSize_SizeRangeCount_0-10": 1, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 260, "FlushTime_max": 260, "FlushTime_mean": 260, "FlushTime_25th_percentile": 260, "FlushTime_median": 260, "FlushTime_75th_percentile": 260, "FlushTime_90th_percentile": 260, "FlushTime_95th_percentile": 260, "FlushTime_98th_percentile": 260, "FlushTime_99th_percentile": 260, "FlushTime_99.9th_percentile": 260, "FlushTime_TimeRangeCount_0-1": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 9, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "ScanSize_SizeRangeCount_0-10": 9, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-12T18:14:51,099 DEBUG [RS:1;9911683f163c:34847 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/oldWALs 2024-11-12T18:14:51,099 INFO [RS:1;9911683f163c:34847 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9911683f163c%2C34847%2C1731435282512:(num 1731435284060) 2024-11-12T18:14:51,099 DEBUG [RS:1;9911683f163c:34847 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:51,099 INFO [RS:1;9911683f163c:34847 {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T18:14:51,099 INFO [RS:1;9911683f163c:34847 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T18:14:51,100 INFO [RS:1;9911683f163c:34847 {}] hbase.ChoreService(370): Chore service for: regionserver/9911683f163c:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-12T18:14:51,100 INFO [RS:1;9911683f163c:34847 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T18:14:51,100 INFO [regionserver/9911683f163c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T18:14:51,100 INFO [RS:1;9911683f163c:34847 {}] executor.ExecutorService(108): Executor-11-RS_OPEN_REGION-regionserver/9911683f163c:0 had [Event #39 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler), Event #40 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler)] on shutdown 2024-11-12T18:14:51,100 INFO [RS:1;9911683f163c:34847 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:34847 2024-11-12T18:14:51,100 WARN [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] util.Threads(127): sleep interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2374) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:51,100 WARN [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,ppp,1731435284929.9fbe24cda440bbf727cd581ab33ad7a9., aborting... java.io.IOException: Failed to report opened region to master: test,ppp,1731435284929.9fbe24cda440bbf727cd581ab33ad7a9. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:51,101 ERROR [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2440): ***** ABORTING region server 9911683f163c,34847,1731435282512: Failed to open region test,ppp,1731435284929.9fbe24cda440bbf727cd581ab33ad7a9. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,ppp,1731435284929.9fbe24cda440bbf727cd581ab33ad7a9. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:51,101 ERROR [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-12T18:14:51,101 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-12T18:14:51,102 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-12T18:14:51,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34847-0x1003533c26e0002, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9911683f163c,34847,1731435282512 2024-11-12T18:14:51,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T18:14:51,103 INFO [RS:1;9911683f163c:34847 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T18:14:51,103 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-12T18:14:51,103 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9911683f163c,34847,1731435282512] 2024-11-12T18:14:51,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T18:14:51,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45243-0x1003533c26e0003, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T18:14:51,104 INFO [RS:2;9911683f163c:45243 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T18:14:51,104 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-12T18:14:51,104 INFO [RS:2;9911683f163c:45243 {}] regionserver.HRegionServer(1031): Exiting; stopping=9911683f163c,45243,1731435282573; zookeeper connection closed. 2024-11-12T18:14:51,104 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@485653be {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@485653be 2024-11-12T18:14:51,105 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9911683f163c,34847,1731435282512 already deleted, retry=false 2024-11-12T18:14:51,105 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9911683f163c,34847,1731435282512 expired; onlineServers=0 2024-11-12T18:14:51,105 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '9911683f163c,36779,1731435281367' ***** 2024-11-12T18:14:51,105 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-12T18:14:51,105 INFO [M:0;9911683f163c:36779 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T18:14:51,105 INFO [M:0;9911683f163c:36779 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T18:14:51,106 DEBUG [M:0;9911683f163c:36779 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-12T18:14:51,106 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-12T18:14:51,106 DEBUG [M:0;9911683f163c:36779 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-12T18:14:51,106 DEBUG [master/9911683f163c:0:becomeActiveMaster-HFileCleaner.large.0-1731435283720 {}] cleaner.HFileCleaner(306): Exit Thread[master/9911683f163c:0:becomeActiveMaster-HFileCleaner.large.0-1731435283720,5,FailOnTimeoutGroup] 2024-11-12T18:14:51,106 INFO [M:0;9911683f163c:36779 {}] hbase.ChoreService(370): Chore service for: master/9911683f163c:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-12T18:14:51,106 INFO [M:0;9911683f163c:36779 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T18:14:51,106 DEBUG [M:0;9911683f163c:36779 {}] master.HMaster(1795): Stopping service threads 2024-11-12T18:14:51,106 INFO [M:0;9911683f163c:36779 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-12T18:14:51,106 DEBUG [master/9911683f163c:0:becomeActiveMaster-HFileCleaner.small.0-1731435283723 {}] cleaner.HFileCleaner(306): Exit Thread[master/9911683f163c:0:becomeActiveMaster-HFileCleaner.small.0-1731435283723,5,FailOnTimeoutGroup] 2024-11-12T18:14:51,107 INFO [M:0;9911683f163c:36779 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:14:51,107 INFO [M:0;9911683f163c:36779 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T18:14:51,107 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-12T18:14:51,108 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 507014848 }, "NonHeapMemoryUsage": { "committed": 163905536, "init": 7667712, "max": -1, "used": 161330568 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "9911683f163c", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 39109, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 115, "ProcessCallTime_min": 0, "ProcessCallTime_max": 20, "ProcessCallTime_mean": 2, "ProcessCallTime_25th_percentile": 5, "ProcessCallTime_median": 10, "ProcessCallTime_75th_percentile": 15, "ProcessCallTime_90th_percentile": 18, "ProcessCallTime_95th_percentile": 19, "ProcessCallTime_98th_percentile": 19, "ProcessCallTime_99th_percentile": 19, "ProcessCallTime_99.9th_percentile": 19, "ProcessCallTime_TimeRangeCount_0-1": 115, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 115, "QueueCallTime_min": 0, "QueueCallTime_max": 12, "QueueCallTime_mean": 1, "QueueCallTime_25th_percentile": 3, "QueueCallTime_median": 6, "QueueCallTime_75th_percentile": 9, "QueueCallTime_90th_percentile": 10, "QueueCallTime_95th_percentile": 11, "QueueCallTime_98th_percentile": 11, "QueueCallTime_99th_percentile": 11, "QueueCallTime_99.9th_percentile": 11, "QueueCallTime_TimeRangeCount_0-1": 115, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 115, "TotalCallTime_min": 0, "TotalCallTime_max": 20, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 5, "TotalCallTime_median": 10, "TotalCallTime_75th_percentile": 15, "TotalCallTime_90th_percentile": 18, "TotalCallTime_95th_percentile": 19, "TotalCallTime_98th_percentile": 19, "TotalCallTime_99th_percentile": 19, "TotalCallTime_99.9th_percentile": 19, "TotalCallTime_TimeRangeCount_0-1": 115, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 115, "ResponseSize_min": 0, "ResponseSize_max": 22359, "ResponseSize_mean": 594, "ResponseSize_25th_percentile": 5589, "ResponseSize_median": 11179, "ResponseSize_75th_percentile": 16769, "ResponseSize_90th_percentile": 20123, "ResponseSize_95th_percentile": 21241, "ResponseSize_98th_percentile": 21911, "ResponseSize_99th_percentile": 22135, "ResponseSize_99.9th_percentile": 22336, "ResponseSize_SizeRangeCount_0-10": 115, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 115, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 308, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 115, "sentBytes": 54885 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "9911683f163c", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:49826", "tag.serverName": "9911683f163c,34847,1731435282512", "tag.clusterId": "8ba834a2-b172-4942-909b-d8381e92fea0", "tag.Context": "regionserver", "tag.Hostname": "9911683f163c", "regionCount": 11, "storeCount": 9, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 2304, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731435282512, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 3341, "localBytesRead": 3341, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.8, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 7474, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 17, "totalRowActionRequestCount": 9, "readRequestCount": 9, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 9, "rpcScanRequestCount": 9, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 8, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 9, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 9, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 8, "Put_min": 0, "Put_max": 3, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 1, "Put_75th_percentile": 2, "Put_90th_percentile": 2, "Put_95th_percentile": 2, "Put_98th_percentile": 2, "Put_99th_percentile": 2, "Put_99.9th_percentile": 2, "Put_TimeRangeCount_0-1": 8, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 46703, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 30111, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 46703, "FlushOutputSize_max": 46703, "FlushOutputSize_mean": 46703, "FlushOutputSize_25th_percentile": 46703, "FlushOutputSize_median": 46703, "FlushOutputSize_75th_percentile": 46703, "FlushOutputSize_90th_percentile": 46703, "FlushOutputSize_95th_percentile": 46703, "FlushOutputSize_98th_percentile": 46703, "FlushOutputSize_99th_percentile": 46703, "FlushOutputSize_99.9th_percentile": 46703, "FlushOutputSize_SizeRangeCount_0-10": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 30111, "FlushMemstoreSize_max": 30111, "FlushMemstoreSize_mean": 30111, "FlushMemstoreSize_25th_percentile": 30111, "FlushMemstoreSize_median": 30111, "FlushMemstoreSize_75th_percentile": 30111, "FlushMemstoreSize_90th_percentile": 30111, "FlushMemstoreSize_95th_percentile": 30111, "FlushMemstoreSize_98th_percentile": 30111, "FlushMemstoreSize_99th_percentile": 30111, "FlushMemstoreSize_99.9th_percentile": 30111, "FlushMemstoreSize_SizeRangeCount_0-10": 1, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 260, "FlushTime_max": 260, "FlushTime_mean": 260, "FlushTime_25th_percentile": 260, "FlushTime_median": 260, "FlushTime_75th_percentile": 260, "FlushTime_90th_percentile": 260, "FlushTime_95th_percentile": 260, "FlushTime_98th_percentile": 260, "FlushTime_99th_percentile": 260, "FlushTime_99.9th_percentile": 260, "FlushTime_TimeRangeCount_0-1": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 9, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "ScanSize_SizeRangeCount_0-10": 9, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-12T18:14:51,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-12T18:14:51,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T18:14:51,110 DEBUG [M:0;9911683f163c:36779 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false 2024-11-12T18:14:51,110 DEBUG [M:0;9911683f163c:36779 {}] master.ActiveMasterManager(353): master:36779-0x1003533c26e0000, quorum=127.0.0.1:49826, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master 2024-11-12T18:14:51,111 INFO [M:0;9911683f163c:36779 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/.lastflushedseqids 2024-11-12T18:14:51,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741870_1046 (size=1) 2024-11-12T18:14:51,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46767 is added to blk_1073741870_1046 (size=1) 2024-11-12T18:14:51,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741870_1046 (size=1) 2024-11-12T18:14:51,129 INFO [M:0;9911683f163c:36779 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T18:14:51,129 INFO [M:0;9911683f163c:36779 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-12T18:14:51,130 DEBUG [M:0;9911683f163c:36779 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T18:14:51,130 INFO [M:0;9911683f163c:36779 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T18:14:51,130 DEBUG [M:0;9911683f163c:36779 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T18:14:51,130 DEBUG [M:0;9911683f163c:36779 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T18:14:51,130 DEBUG [M:0;9911683f163c:36779 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T18:14:51,130 INFO [M:0;9911683f163c:36779 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=259.20 KB heapSize=315.03 KB 2024-11-12T18:14:51,149 DEBUG [M:0;9911683f163c:36779 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ac05f4e970b34509be30682ee8ab8309 is 82, key is hbase:meta,,1/info:regioninfo/1731435289387/Put/seqid=0 2024-11-12T18:14:51,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42977 is added to blk_1073741871_1047 (size=6071) 2024-11-12T18:14:51,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35529 is added to blk_1073741871_1047 (size=6071) 2024-11-12T18:14:51,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46767 is added to blk_1073741871_1047 (size=6071) 2024-11-12T18:14:51,163 INFO [M:0;9911683f163c:36779 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=598 (bloomFilter=true), to=hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ac05f4e970b34509be30682ee8ab8309 2024-11-12T18:14:51,189 DEBUG [M:0;9911683f163c:36779 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38437/user/jenkins/test-data/b3ae2c86-6b0e-cdb3-b4e0-5d0c6577212d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e43da82b76724d5b892aa637e4e382c9 is 2990, key is \x00\x00\x00\x00\x00\x00\x00