2024-11-08 19:13:14,680 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-08 19:13:14,704 main DEBUG Took 0.020965 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-08 19:13:14,704 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-08 19:13:14,705 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-08 19:13:14,706 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-08 19:13:14,707 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 19:13:14,715 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-08 19:13:14,728 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 19:13:14,730 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 19:13:14,730 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 19:13:14,731 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 19:13:14,732 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 19:13:14,732 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 19:13:14,733 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 19:13:14,734 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 19:13:14,735 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 19:13:14,735 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 19:13:14,736 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 19:13:14,737 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 19:13:14,737 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 19:13:14,738 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 19:13:14,739 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 19:13:14,739 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 19:13:14,740 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 19:13:14,741 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 19:13:14,741 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 19:13:14,742 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 19:13:14,742 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 19:13:14,743 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 19:13:14,744 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 19:13:14,744 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 19:13:14,745 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 19:13:14,745 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-08 19:13:14,747 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 19:13:14,749 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-08 19:13:14,751 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-08 19:13:14,752 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-08 19:13:14,754 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-08 19:13:14,755 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-08 19:13:14,763 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-08 19:13:14,766 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-08 19:13:14,768 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-08 19:13:14,768 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-08 19:13:14,768 main DEBUG createAppenders(={Console}) 2024-11-08 19:13:14,769 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-11-08 19:13:14,770 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-08 19:13:14,770 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-11-08 19:13:14,771 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-08 19:13:14,771 main DEBUG OutputStream closed 2024-11-08 19:13:14,771 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-08 19:13:14,771 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-08 19:13:14,772 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-11-08 19:13:14,864 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-08 19:13:14,866 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-08 19:13:14,868 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-08 19:13:14,869 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-08 19:13:14,870 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-08 19:13:14,871 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-08 19:13:14,871 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-08 19:13:14,872 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-08 19:13:14,872 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-08 19:13:14,873 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-08 19:13:14,874 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-08 19:13:14,874 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-08 19:13:14,875 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-08 19:13:14,875 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-08 19:13:14,876 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-08 19:13:14,876 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-08 19:13:14,877 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-08 19:13:14,878 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-08 19:13:14,881 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08 19:13:14,881 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-11-08 19:13:14,882 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-08 19:13:14,883 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-11-08T19:13:14,903 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-11-08 19:13:14,907 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-08 19:13:14,907 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08T19:13:15,174 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd 2024-11-08T19:13:15,197 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-08T19:13:15,218 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/cluster_79c0c4f2-2bfc-2c08-96de-113f2fd3fb9c, deleteOnExit=true 2024-11-08T19:13:15,219 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-08T19:13:15,220 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/test.cache.data in system properties and HBase conf 2024-11-08T19:13:15,222 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/hadoop.tmp.dir in system properties and HBase conf 2024-11-08T19:13:15,223 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/hadoop.log.dir in system properties and HBase conf 2024-11-08T19:13:15,224 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-08T19:13:15,225 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-08T19:13:15,226 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-08T19:13:15,344 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-08T19:13:15,434 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T19:13:15,439 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-08T19:13:15,439 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-08T19:13:15,440 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-08T19:13:15,441 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-08T19:13:15,441 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-08T19:13:15,442 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-08T19:13:15,442 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-08T19:13:15,443 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-08T19:13:15,443 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-08T19:13:15,444 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/nfs.dump.dir in system properties and HBase conf 2024-11-08T19:13:15,444 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/java.io.tmpdir in system properties and HBase conf 2024-11-08T19:13:15,445 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-08T19:13:15,445 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-08T19:13:15,446 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-08T19:13:16,882 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-08T19:13:16,977 INFO [Time-limited test {}] log.Log(170): Logging initialized @2999ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-08T19:13:17,046 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T19:13:17,116 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T19:13:17,140 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T19:13:17,140 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T19:13:17,142 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-08T19:13:17,155 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T19:13:17,160 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/hadoop.log.dir/,AVAILABLE} 2024-11-08T19:13:17,161 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T19:13:17,359 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64418798{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/java.io.tmpdir/jetty-localhost-33911-hadoop-hdfs-3_4_1-tests_jar-_-any-12004116684292814340/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-08T19:13:17,369 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:33911} 2024-11-08T19:13:17,369 INFO [Time-limited test {}] server.Server(415): Started @3392ms 2024-11-08T19:13:18,122 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T19:13:18,132 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T19:13:18,133 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T19:13:18,134 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T19:13:18,134 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-08T19:13:18,135 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3a300759{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/hadoop.log.dir/,AVAILABLE} 2024-11-08T19:13:18,135 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@474d518f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T19:13:18,262 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@79b30fb0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/java.io.tmpdir/jetty-localhost-35351-hadoop-hdfs-3_4_1-tests_jar-_-any-3668619645998362710/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T19:13:18,262 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@548cd725{HTTP/1.1, (http/1.1)}{localhost:35351} 2024-11-08T19:13:18,263 INFO [Time-limited test {}] server.Server(415): Started @4286ms 2024-11-08T19:13:18,311 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-08T19:13:18,425 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T19:13:18,433 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T19:13:18,434 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T19:13:18,434 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T19:13:18,435 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-08T19:13:18,436 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@29652cf4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/hadoop.log.dir/,AVAILABLE} 2024-11-08T19:13:18,437 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1b0a32ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T19:13:18,568 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@34fcb585{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/java.io.tmpdir/jetty-localhost-42959-hadoop-hdfs-3_4_1-tests_jar-_-any-2694071650843864932/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T19:13:18,569 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@389bb6f7{HTTP/1.1, (http/1.1)}{localhost:42959} 2024-11-08T19:13:18,569 INFO [Time-limited test {}] server.Server(415): Started @4592ms 2024-11-08T19:13:18,571 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-08T19:13:18,606 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T19:13:18,610 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T19:13:18,611 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T19:13:18,611 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T19:13:18,612 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-08T19:13:18,612 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@290e2eec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/hadoop.log.dir/,AVAILABLE} 2024-11-08T19:13:18,613 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@417efacb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T19:13:18,728 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3c1f701{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/java.io.tmpdir/jetty-localhost-36885-hadoop-hdfs-3_4_1-tests_jar-_-any-11026349128851733758/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T19:13:18,730 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3d599bb4{HTTP/1.1, (http/1.1)}{localhost:36885} 2024-11-08T19:13:18,730 INFO [Time-limited test {}] server.Server(415): Started @4753ms 2024-11-08T19:13:18,733 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-08T19:13:20,374 WARN [Thread-120 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/cluster_79c0c4f2-2bfc-2c08-96de-113f2fd3fb9c/data/data1/current/BP-585874096-172.17.0.2-1731093195986/current, will proceed with Du for space computation calculation, 2024-11-08T19:13:20,374 WARN [Thread-121 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/cluster_79c0c4f2-2bfc-2c08-96de-113f2fd3fb9c/data/data2/current/BP-585874096-172.17.0.2-1731093195986/current, will proceed with Du for space computation calculation, 2024-11-08T19:13:20,384 WARN [Thread-126 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/cluster_79c0c4f2-2bfc-2c08-96de-113f2fd3fb9c/data/data3/current/BP-585874096-172.17.0.2-1731093195986/current, will proceed with Du for space computation calculation, 2024-11-08T19:13:20,384 WARN [Thread-127 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/cluster_79c0c4f2-2bfc-2c08-96de-113f2fd3fb9c/data/data4/current/BP-585874096-172.17.0.2-1731093195986/current, will proceed with Du for space computation calculation, 2024-11-08T19:13:20,409 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-08T19:13:20,409 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-08T19:13:20,460 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf45bece9163b0f49 with lease ID 0xb09a0b97b2d5fc62: Processing first storage report for DS-5e1bf434-3af5-4fc9-abe7-9a902de021b6 from datanode DatanodeRegistration(127.0.0.1:46657, datanodeUuid=df5501d9-e532-471c-b303-aebd945d69ac, infoPort=36293, infoSecurePort=0, ipcPort=40119, storageInfo=lv=-57;cid=testClusterID;nsid=1727487417;c=1731093195986) 2024-11-08T19:13:20,461 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf45bece9163b0f49 with lease ID 0xb09a0b97b2d5fc62: from storage DS-5e1bf434-3af5-4fc9-abe7-9a902de021b6 node DatanodeRegistration(127.0.0.1:46657, datanodeUuid=df5501d9-e532-471c-b303-aebd945d69ac, infoPort=36293, infoSecurePort=0, ipcPort=40119, storageInfo=lv=-57;cid=testClusterID;nsid=1727487417;c=1731093195986), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-08T19:13:20,462 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc7a94c7f9cf1274 with lease ID 0xb09a0b97b2d5fc63: Processing first storage report for DS-c0e88d76-cfe1-4c35-bbea-3e8b1423e597 from datanode DatanodeRegistration(127.0.0.1:42249, datanodeUuid=3a969f21-2fc4-4416-ac5c-2994a3eb4f53, infoPort=46659, infoSecurePort=0, ipcPort=41111, storageInfo=lv=-57;cid=testClusterID;nsid=1727487417;c=1731093195986) 2024-11-08T19:13:20,462 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc7a94c7f9cf1274 with lease ID 0xb09a0b97b2d5fc63: from storage DS-c0e88d76-cfe1-4c35-bbea-3e8b1423e597 node DatanodeRegistration(127.0.0.1:42249, datanodeUuid=3a969f21-2fc4-4416-ac5c-2994a3eb4f53, infoPort=46659, infoSecurePort=0, ipcPort=41111, storageInfo=lv=-57;cid=testClusterID;nsid=1727487417;c=1731093195986), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-08T19:13:20,462 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf45bece9163b0f49 with lease ID 0xb09a0b97b2d5fc62: Processing first storage report for DS-37bf3fc7-7967-42ee-a759-1f4f9565c954 from datanode DatanodeRegistration(127.0.0.1:46657, datanodeUuid=df5501d9-e532-471c-b303-aebd945d69ac, infoPort=36293, infoSecurePort=0, ipcPort=40119, storageInfo=lv=-57;cid=testClusterID;nsid=1727487417;c=1731093195986) 2024-11-08T19:13:20,463 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf45bece9163b0f49 with lease ID 0xb09a0b97b2d5fc62: from storage DS-37bf3fc7-7967-42ee-a759-1f4f9565c954 node DatanodeRegistration(127.0.0.1:46657, datanodeUuid=df5501d9-e532-471c-b303-aebd945d69ac, infoPort=36293, infoSecurePort=0, ipcPort=40119, storageInfo=lv=-57;cid=testClusterID;nsid=1727487417;c=1731093195986), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-08T19:13:20,463 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc7a94c7f9cf1274 with lease ID 0xb09a0b97b2d5fc63: Processing first storage report for DS-72e73c10-d187-4dd9-95fc-fabb83a97e99 from datanode DatanodeRegistration(127.0.0.1:42249, datanodeUuid=3a969f21-2fc4-4416-ac5c-2994a3eb4f53, infoPort=46659, infoSecurePort=0, ipcPort=41111, storageInfo=lv=-57;cid=testClusterID;nsid=1727487417;c=1731093195986) 2024-11-08T19:13:20,463 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc7a94c7f9cf1274 with lease ID 0xb09a0b97b2d5fc63: from storage DS-72e73c10-d187-4dd9-95fc-fabb83a97e99 node DatanodeRegistration(127.0.0.1:42249, datanodeUuid=3a969f21-2fc4-4416-ac5c-2994a3eb4f53, infoPort=46659, infoSecurePort=0, ipcPort=41111, storageInfo=lv=-57;cid=testClusterID;nsid=1727487417;c=1731093195986), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-08T19:13:20,539 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/cluster_79c0c4f2-2bfc-2c08-96de-113f2fd3fb9c/data/data5/current/BP-585874096-172.17.0.2-1731093195986/current, will proceed with Du for space computation calculation, 2024-11-08T19:13:20,539 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/cluster_79c0c4f2-2bfc-2c08-96de-113f2fd3fb9c/data/data6/current/BP-585874096-172.17.0.2-1731093195986/current, will proceed with Du for space computation calculation, 2024-11-08T19:13:20,557 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-08T19:13:20,561 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9d6b6e494811b2d1 with lease ID 0xb09a0b97b2d5fc64: Processing first storage report for DS-6e01471a-b8b4-43a1-88b3-b57bf9ad1710 from datanode DatanodeRegistration(127.0.0.1:40347, datanodeUuid=e5fad3ad-0de2-4a49-8a13-2c9609c5c7b8, infoPort=39837, infoSecurePort=0, ipcPort=33635, storageInfo=lv=-57;cid=testClusterID;nsid=1727487417;c=1731093195986) 2024-11-08T19:13:20,561 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9d6b6e494811b2d1 with lease ID 0xb09a0b97b2d5fc64: from storage DS-6e01471a-b8b4-43a1-88b3-b57bf9ad1710 node DatanodeRegistration(127.0.0.1:40347, datanodeUuid=e5fad3ad-0de2-4a49-8a13-2c9609c5c7b8, infoPort=39837, infoSecurePort=0, ipcPort=33635, storageInfo=lv=-57;cid=testClusterID;nsid=1727487417;c=1731093195986), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-08T19:13:20,561 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9d6b6e494811b2d1 with lease ID 0xb09a0b97b2d5fc64: Processing first storage report for DS-bc72a04f-18ca-4250-9f8e-0f3164a39797 from datanode DatanodeRegistration(127.0.0.1:40347, datanodeUuid=e5fad3ad-0de2-4a49-8a13-2c9609c5c7b8, infoPort=39837, infoSecurePort=0, ipcPort=33635, storageInfo=lv=-57;cid=testClusterID;nsid=1727487417;c=1731093195986) 2024-11-08T19:13:20,561 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9d6b6e494811b2d1 with lease ID 0xb09a0b97b2d5fc64: from storage DS-bc72a04f-18ca-4250-9f8e-0f3164a39797 node DatanodeRegistration(127.0.0.1:40347, datanodeUuid=e5fad3ad-0de2-4a49-8a13-2c9609c5c7b8, infoPort=39837, infoSecurePort=0, ipcPort=33635, storageInfo=lv=-57;cid=testClusterID;nsid=1727487417;c=1731093195986), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-08T19:13:20,666 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd 2024-11-08T19:13:20,744 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/cluster_79c0c4f2-2bfc-2c08-96de-113f2fd3fb9c/zookeeper_0, clientPort=56395, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/cluster_79c0c4f2-2bfc-2c08-96de-113f2fd3fb9c/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/cluster_79c0c4f2-2bfc-2c08-96de-113f2fd3fb9c/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-08T19:13:20,755 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56395 2024-11-08T19:13:20,771 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T19:13:20,775 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T19:13:20,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741825_1001 (size=7) 2024-11-08T19:13:20,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741825_1001 (size=7) 2024-11-08T19:13:20,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741825_1001 (size=7) 2024-11-08T19:13:21,388 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f with version=8 2024-11-08T19:13:21,389 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/hbase-staging 2024-11-08T19:13:21,495 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-08T19:13:21,789 INFO [Time-limited test {}] client.ConnectionUtils(128): master/692b295ad45c:0 server-side Connection retries=45 2024-11-08T19:13:21,804 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T19:13:21,805 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T19:13:21,812 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T19:13:21,812 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T19:13:21,813 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T19:13:22,011 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-08T19:13:22,068 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-08T19:13:22,080 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-08T19:13:22,085 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T19:13:22,115 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 3184 (auto-detected) 2024-11-08T19:13:22,117 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-08T19:13:22,146 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39509 2024-11-08T19:13:22,166 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39509 connecting to ZooKeeper ensemble=127.0.0.1:56395 2024-11-08T19:13:22,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:395090x0, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T19:13:22,340 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39509-0x1011bddfb000000 connected 2024-11-08T19:13:22,499 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T19:13:22,503 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T19:13:22,543 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T19:13:22,549 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f, hbase.cluster.distributed=false 2024-11-08T19:13:22,584 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T19:13:22,614 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39509 2024-11-08T19:13:22,615 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39509 2024-11-08T19:13:22,616 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39509 2024-11-08T19:13:22,627 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39509 2024-11-08T19:13:22,630 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39509 2024-11-08T19:13:22,795 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/692b295ad45c:0 server-side Connection retries=45 2024-11-08T19:13:22,797 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T19:13:22,797 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T19:13:22,798 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T19:13:22,798 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T19:13:22,798 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T19:13:22,802 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T19:13:22,806 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T19:13:22,808 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36007 2024-11-08T19:13:22,811 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36007 connecting to ZooKeeper ensemble=127.0.0.1:56395 2024-11-08T19:13:22,812 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T19:13:22,819 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T19:13:22,916 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:360070x0, quorum=127.0.0.1:56395, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T19:13:22,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:360070x0, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T19:13:22,918 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36007-0x1011bddfb000001 connected 2024-11-08T19:13:22,922 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T19:13:22,932 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T19:13:22,936 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T19:13:22,943 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T19:13:22,948 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36007 2024-11-08T19:13:22,948 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36007 2024-11-08T19:13:22,952 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36007 2024-11-08T19:13:22,957 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36007 2024-11-08T19:13:22,958 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36007 2024-11-08T19:13:22,978 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/692b295ad45c:0 server-side Connection retries=45 2024-11-08T19:13:22,978 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T19:13:22,978 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T19:13:22,979 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T19:13:22,979 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T19:13:22,979 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T19:13:22,979 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T19:13:22,980 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T19:13:22,981 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42693 2024-11-08T19:13:22,983 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42693 connecting to ZooKeeper ensemble=127.0.0.1:56395 2024-11-08T19:13:22,984 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T19:13:22,989 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T19:13:23,014 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:426930x0, quorum=127.0.0.1:56395, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T19:13:23,014 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:426930x0, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T19:13:23,015 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T19:13:23,019 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42693-0x1011bddfb000002 connected 2024-11-08T19:13:23,020 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T19:13:23,022 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T19:13:23,026 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T19:13:23,032 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42693 2024-11-08T19:13:23,033 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42693 2024-11-08T19:13:23,034 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42693 2024-11-08T19:13:23,037 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42693 2024-11-08T19:13:23,037 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42693 2024-11-08T19:13:23,062 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/692b295ad45c:0 server-side Connection retries=45 2024-11-08T19:13:23,063 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T19:13:23,063 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T19:13:23,063 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T19:13:23,064 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T19:13:23,064 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T19:13:23,064 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T19:13:23,065 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T19:13:23,066 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39647 2024-11-08T19:13:23,069 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39647 connecting to ZooKeeper ensemble=127.0.0.1:56395 2024-11-08T19:13:23,070 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T19:13:23,075 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T19:13:23,094 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:396470x0, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T19:13:23,095 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:396470x0, quorum=127.0.0.1:56395, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T19:13:23,096 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T19:13:23,096 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39647-0x1011bddfb000003 connected 2024-11-08T19:13:23,103 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T19:13:23,104 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39647-0x1011bddfb000003, quorum=127.0.0.1:56395, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T19:13:23,107 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39647-0x1011bddfb000003, quorum=127.0.0.1:56395, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T19:13:23,112 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39647 2024-11-08T19:13:23,113 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39647 2024-11-08T19:13:23,116 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39647 2024-11-08T19:13:23,119 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39647 2024-11-08T19:13:23,120 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39647 2024-11-08T19:13:23,145 DEBUG [M:0;692b295ad45c:39509 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;692b295ad45c:39509 2024-11-08T19:13:23,148 INFO [master/692b295ad45c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/692b295ad45c,39509,1731093201567 2024-11-08T19:13:23,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T19:13:23,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39647-0x1011bddfb000003, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T19:13:23,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T19:13:23,160 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T19:13:23,164 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/692b295ad45c,39509,1731093201567 2024-11-08T19:13:23,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T19:13:23,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T19:13:23,231 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T19:13:23,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T19:13:23,231 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T19:13:23,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39647-0x1011bddfb000003, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T19:13:23,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39647-0x1011bddfb000003, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T19:13:23,233 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-08T19:13:23,235 INFO [master/692b295ad45c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/692b295ad45c,39509,1731093201567 from backup master directory 2024-11-08T19:13:23,322 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T19:13:23,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/692b295ad45c,39509,1731093201567 2024-11-08T19:13:23,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39647-0x1011bddfb000003, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T19:13:23,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T19:13:23,323 WARN [master/692b295ad45c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T19:13:23,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T19:13:23,324 INFO [master/692b295ad45c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=692b295ad45c,39509,1731093201567 2024-11-08T19:13:23,326 INFO [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-08T19:13:23,328 INFO [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-08T19:13:23,389 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/hbase.id] with ID: c2739604-5628-4fa2-a3e8-33881b328b44 2024-11-08T19:13:23,389 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/.tmp/hbase.id 2024-11-08T19:13:23,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741826_1002 (size=42) 2024-11-08T19:13:23,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741826_1002 (size=42) 2024-11-08T19:13:23,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741826_1002 (size=42) 2024-11-08T19:13:23,419 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/.tmp/hbase.id]:[hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/hbase.id] 2024-11-08T19:13:23,472 INFO [master/692b295ad45c:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T19:13:23,477 INFO [master/692b295ad45c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-08T19:13:23,503 INFO [master/692b295ad45c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 23ms. 2024-11-08T19:13:23,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T19:13:23,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39647-0x1011bddfb000003, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T19:13:23,517 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T19:13:23,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T19:13:23,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741827_1003 (size=196) 2024-11-08T19:13:23,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741827_1003 (size=196) 2024-11-08T19:13:23,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741827_1003 (size=196) 2024-11-08T19:13:23,567 INFO [master/692b295ad45c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T19:13:23,569 INFO [master/692b295ad45c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-08T19:13:23,585 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T19:13:23,589 INFO [master/692b295ad45c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T19:13:23,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741828_1004 (size=1189) 2024-11-08T19:13:23,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741828_1004 (size=1189) 2024-11-08T19:13:23,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741828_1004 (size=1189) 2024-11-08T19:13:23,640 INFO [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/MasterData/data/master/store 2024-11-08T19:13:23,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741829_1005 (size=34) 2024-11-08T19:13:23,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741829_1005 (size=34) 2024-11-08T19:13:23,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741829_1005 (size=34) 2024-11-08T19:13:23,666 INFO [master/692b295ad45c:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-08T19:13:23,669 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:23,670 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T19:13:23,671 INFO [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T19:13:23,671 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T19:13:23,672 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T19:13:23,672 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T19:13:23,673 INFO [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T19:13:23,674 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731093203670Disabling compacts and flushes for region at 1731093203670Disabling writes for close at 1731093203672 (+2 ms)Writing region close event to WAL at 1731093203673 (+1 ms)Closed at 1731093203673 2024-11-08T19:13:23,676 WARN [master/692b295ad45c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/MasterData/data/master/store/.initializing 2024-11-08T19:13:23,676 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/MasterData/WALs/692b295ad45c,39509,1731093201567 2024-11-08T19:13:23,684 INFO [master/692b295ad45c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T19:13:23,697 INFO [master/692b295ad45c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=692b295ad45c%2C39509%2C1731093201567, suffix=, logDir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/MasterData/WALs/692b295ad45c,39509,1731093201567, archiveDir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/MasterData/oldWALs, maxLogs=10 2024-11-08T19:13:23,720 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/MasterData/WALs/692b295ad45c,39509,1731093201567/692b295ad45c%2C39509%2C1731093201567.1731093203702, exclude list is [], retry=0 2024-11-08T19:13:23,738 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42249,DS-c0e88d76-cfe1-4c35-bbea-3e8b1423e597,DISK] 2024-11-08T19:13:23,738 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40347,DS-6e01471a-b8b4-43a1-88b3-b57bf9ad1710,DISK] 2024-11-08T19:13:23,738 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46657,DS-5e1bf434-3af5-4fc9-abe7-9a902de021b6,DISK] 2024-11-08T19:13:23,741 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-08T19:13:23,779 INFO [master/692b295ad45c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/MasterData/WALs/692b295ad45c,39509,1731093201567/692b295ad45c%2C39509%2C1731093201567.1731093203702 2024-11-08T19:13:23,780 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36293:36293),(127.0.0.1/127.0.0.1:39837:39837),(127.0.0.1/127.0.0.1:46659:46659)] 2024-11-08T19:13:23,780 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-08T19:13:23,781 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:23,784 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T19:13:23,785 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T19:13:23,825 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T19:13:23,848 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-08T19:13:23,852 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:23,854 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T19:13:23,855 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T19:13:23,859 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-08T19:13:23,859 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:23,860 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T19:13:23,860 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T19:13:23,864 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-08T19:13:23,864 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:23,865 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T19:13:23,866 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T19:13:23,869 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-08T19:13:23,869 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:23,870 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T19:13:23,870 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T19:13:23,874 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T19:13:23,875 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T19:13:23,879 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T19:13:23,880 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T19:13:23,883 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T19:13:23,887 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T19:13:23,891 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T19:13:23,892 INFO [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59287878, jitterRate=-0.1165417730808258}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T19:13:23,899 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731093203800Initializing all the Stores at 1731093203802 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731093203803 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731093203803Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731093203804 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731093203804Cleaning up temporary data from old regions at 1731093203880 (+76 ms)Region opened successfully at 1731093203898 (+18 ms) 2024-11-08T19:13:23,900 INFO [master/692b295ad45c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-08T19:13:23,930 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4790aa43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=692b295ad45c/172.17.0.2:0 2024-11-08T19:13:23,958 INFO [master/692b295ad45c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-08T19:13:23,968 INFO [master/692b295ad45c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T19:13:23,968 INFO [master/692b295ad45c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T19:13:23,970 INFO [master/692b295ad45c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T19:13:23,972 INFO [master/692b295ad45c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-08T19:13:23,976 INFO [master/692b295ad45c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-08T19:13:23,977 INFO [master/692b295ad45c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-08T19:13:24,000 INFO [master/692b295ad45c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-08T19:13:24,008 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-08T19:13:24,062 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-08T19:13:24,065 INFO [master/692b295ad45c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-08T19:13:24,068 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-08T19:13:24,074 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-08T19:13:24,077 INFO [master/692b295ad45c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-08T19:13:24,082 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-08T19:13:24,093 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-08T19:13:24,095 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-08T19:13:24,106 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-08T19:13:24,129 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-08T19:13:24,137 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-08T19:13:24,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T19:13:24,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39647-0x1011bddfb000003, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T19:13:24,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T19:13:24,148 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T19:13:24,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T19:13:24,149 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T19:13:24,149 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39647-0x1011bddfb000003, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T19:13:24,149 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T19:13:24,152 INFO [master/692b295ad45c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=692b295ad45c,39509,1731093201567, sessionid=0x1011bddfb000000, setting cluster-up flag (Was=false) 2024-11-08T19:13:24,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39647-0x1011bddfb000003, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T19:13:24,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T19:13:24,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T19:13:24,180 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T19:13:24,212 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-08T19:13:24,217 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=692b295ad45c,39509,1731093201567 2024-11-08T19:13:24,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39647-0x1011bddfb000003, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T19:13:24,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T19:13:24,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T19:13:24,244 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T19:13:24,274 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-08T19:13:24,278 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=692b295ad45c,39509,1731093201567 2024-11-08T19:13:24,286 INFO [master/692b295ad45c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-08T19:13:24,328 INFO [RS:0;692b295ad45c:36007 {}] regionserver.HRegionServer(746): ClusterId : c2739604-5628-4fa2-a3e8-33881b328b44 2024-11-08T19:13:24,328 INFO [RS:2;692b295ad45c:39647 {}] regionserver.HRegionServer(746): ClusterId : c2739604-5628-4fa2-a3e8-33881b328b44 2024-11-08T19:13:24,328 INFO [RS:1;692b295ad45c:42693 {}] regionserver.HRegionServer(746): ClusterId : c2739604-5628-4fa2-a3e8-33881b328b44 2024-11-08T19:13:24,330 DEBUG [RS:1;692b295ad45c:42693 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T19:13:24,330 DEBUG [RS:2;692b295ad45c:39647 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T19:13:24,330 DEBUG [RS:0;692b295ad45c:36007 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T19:13:24,350 DEBUG [RS:0;692b295ad45c:36007 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T19:13:24,350 DEBUG [RS:2;692b295ad45c:39647 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T19:13:24,350 DEBUG [RS:1;692b295ad45c:42693 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T19:13:24,350 DEBUG [RS:0;692b295ad45c:36007 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T19:13:24,350 DEBUG [RS:2;692b295ad45c:39647 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T19:13:24,350 DEBUG [RS:1;692b295ad45c:42693 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T19:13:24,353 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T19:13:24,360 DEBUG [RS:1;692b295ad45c:42693 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T19:13:24,360 DEBUG [RS:0;692b295ad45c:36007 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T19:13:24,360 DEBUG [RS:2;692b295ad45c:39647 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T19:13:24,360 DEBUG [RS:1;692b295ad45c:42693 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7264b63c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=692b295ad45c/172.17.0.2:0 2024-11-08T19:13:24,360 DEBUG [RS:0;692b295ad45c:36007 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4308550b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=692b295ad45c/172.17.0.2:0 2024-11-08T19:13:24,361 DEBUG [RS:2;692b295ad45c:39647 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b303e59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=692b295ad45c/172.17.0.2:0 2024-11-08T19:13:24,363 INFO [master/692b295ad45c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-08T19:13:24,371 INFO [master/692b295ad45c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-08T19:13:24,374 DEBUG [RS:1;692b295ad45c:42693 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;692b295ad45c:42693 2024-11-08T19:13:24,376 DEBUG [RS:0;692b295ad45c:36007 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;692b295ad45c:36007 2024-11-08T19:13:24,377 INFO [RS:1;692b295ad45c:42693 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T19:13:24,377 INFO [RS:0;692b295ad45c:36007 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T19:13:24,377 INFO [RS:1;692b295ad45c:42693 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T19:13:24,377 INFO [RS:0;692b295ad45c:36007 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T19:13:24,377 DEBUG [RS:1;692b295ad45c:42693 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T19:13:24,377 DEBUG [RS:0;692b295ad45c:36007 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T19:13:24,379 DEBUG [RS:2;692b295ad45c:39647 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;692b295ad45c:39647 2024-11-08T19:13:24,379 INFO [RS:2;692b295ad45c:39647 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T19:13:24,379 INFO [RS:2;692b295ad45c:39647 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T19:13:24,379 DEBUG [RS:2;692b295ad45c:39647 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T19:13:24,380 INFO [RS:1;692b295ad45c:42693 {}] regionserver.HRegionServer(2659): reportForDuty to master=692b295ad45c,39509,1731093201567 with port=42693, startcode=1731093202977 2024-11-08T19:13:24,380 INFO [RS:0;692b295ad45c:36007 {}] regionserver.HRegionServer(2659): reportForDuty to master=692b295ad45c,39509,1731093201567 with port=36007, startcode=1731093202744 2024-11-08T19:13:24,381 INFO [RS:2;692b295ad45c:39647 {}] regionserver.HRegionServer(2659): reportForDuty to master=692b295ad45c,39509,1731093201567 with port=39647, startcode=1731093203061 2024-11-08T19:13:24,378 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 692b295ad45c,39509,1731093201567 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-08T19:13:24,386 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/692b295ad45c:0, corePoolSize=5, maxPoolSize=5 2024-11-08T19:13:24,386 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/692b295ad45c:0, corePoolSize=5, maxPoolSize=5 2024-11-08T19:13:24,386 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/692b295ad45c:0, corePoolSize=5, maxPoolSize=5 2024-11-08T19:13:24,387 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/692b295ad45c:0, corePoolSize=5, maxPoolSize=5 2024-11-08T19:13:24,387 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/692b295ad45c:0, corePoolSize=10, maxPoolSize=10 2024-11-08T19:13:24,387 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:13:24,387 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/692b295ad45c:0, corePoolSize=2, maxPoolSize=2 2024-11-08T19:13:24,387 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:13:24,392 DEBUG [RS:1;692b295ad45c:42693 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T19:13:24,392 DEBUG [RS:0;692b295ad45c:36007 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T19:13:24,392 DEBUG [RS:2;692b295ad45c:39647 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T19:13:24,395 INFO [master/692b295ad45c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731093234395 2024-11-08T19:13:24,397 INFO [master/692b295ad45c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-08T19:13:24,399 INFO [master/692b295ad45c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-08T19:13:24,401 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T19:13:24,402 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-08T19:13:24,403 INFO [master/692b295ad45c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-08T19:13:24,403 INFO [master/692b295ad45c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-08T19:13:24,403 INFO [master/692b295ad45c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-08T19:13:24,404 INFO [master/692b295ad45c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-08T19:13:24,404 INFO [master/692b295ad45c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T19:13:24,408 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:24,409 INFO [master/692b295ad45c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-08T19:13:24,408 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-08T19:13:24,410 INFO [master/692b295ad45c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-08T19:13:24,411 INFO [master/692b295ad45c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-08T19:13:24,424 INFO [master/692b295ad45c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-08T19:13:24,425 INFO [master/692b295ad45c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-08T19:13:24,430 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/692b295ad45c:0:becomeActiveMaster-HFileCleaner.large.0-1731093204428,5,FailOnTimeoutGroup] 2024-11-08T19:13:24,432 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/692b295ad45c:0:becomeActiveMaster-HFileCleaner.small.0-1731093204430,5,FailOnTimeoutGroup] 2024-11-08T19:13:24,433 INFO [master/692b295ad45c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T19:13:24,433 INFO [master/692b295ad45c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-08T19:13:24,434 INFO [master/692b295ad45c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-08T19:13:24,434 INFO [master/692b295ad45c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-08T19:13:24,471 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54401, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T19:13:24,471 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45193, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T19:13:24,471 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44159, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T19:13:24,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741831_1007 (size=1321) 2024-11-08T19:13:24,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741831_1007 (size=1321) 2024-11-08T19:13:24,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741831_1007 (size=1321) 2024-11-08T19:13:24,479 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-08T19:13:24,479 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39509 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 692b295ad45c,39647,1731093203061 2024-11-08T19:13:24,479 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f 2024-11-08T19:13:24,482 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39509 {}] master.ServerManager(517): Registering regionserver=692b295ad45c,39647,1731093203061 2024-11-08T19:13:24,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741832_1008 (size=32) 2024-11-08T19:13:24,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741832_1008 (size=32) 2024-11-08T19:13:24,506 DEBUG [RS:2;692b295ad45c:39647 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f 2024-11-08T19:13:24,506 DEBUG [RS:2;692b295ad45c:39647 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33621 2024-11-08T19:13:24,507 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39509 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 692b295ad45c,36007,1731093202744 2024-11-08T19:13:24,507 DEBUG [RS:2;692b295ad45c:39647 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T19:13:24,507 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39509 {}] master.ServerManager(517): Registering regionserver=692b295ad45c,36007,1731093202744 2024-11-08T19:13:24,508 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:24,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741832_1008 (size=32) 2024-11-08T19:13:24,511 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39509 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 692b295ad45c,42693,1731093202977 2024-11-08T19:13:24,512 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39509 {}] master.ServerManager(517): Registering regionserver=692b295ad45c,42693,1731093202977 2024-11-08T19:13:24,512 DEBUG [RS:0;692b295ad45c:36007 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f 2024-11-08T19:13:24,512 DEBUG [RS:0;692b295ad45c:36007 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33621 2024-11-08T19:13:24,512 DEBUG [RS:0;692b295ad45c:36007 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T19:13:24,513 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-08T19:13:24,515 DEBUG [RS:1;692b295ad45c:42693 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f 2024-11-08T19:13:24,515 DEBUG [RS:1;692b295ad45c:42693 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33621 2024-11-08T19:13:24,515 DEBUG [RS:1;692b295ad45c:42693 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T19:13:24,516 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T19:13:24,516 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:24,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T19:13:24,517 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T19:13:24,518 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-08T19:13:24,521 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T19:13:24,521 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:24,522 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T19:13:24,522 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-08T19:13:24,525 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T19:13:24,525 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:24,526 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T19:13:24,526 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-08T19:13:24,529 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T19:13:24,529 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:24,530 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T19:13:24,530 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T19:13:24,532 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/hbase/meta/1588230740 2024-11-08T19:13:24,533 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/hbase/meta/1588230740 2024-11-08T19:13:24,536 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T19:13:24,536 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T19:13:24,537 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T19:13:24,540 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T19:13:24,544 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T19:13:24,546 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71062833, jitterRate=0.058918729424476624}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T19:13:24,550 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731093204508Initializing all the Stores at 1731093204512 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731093204512Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731093204512Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731093204513 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731093204513Cleaning up temporary data from old regions at 1731093204536 (+23 ms)Region opened successfully at 1731093204550 (+14 ms) 2024-11-08T19:13:24,550 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T19:13:24,550 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T19:13:24,550 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T19:13:24,550 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T19:13:24,550 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T19:13:24,552 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T19:13:24,552 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731093204550Disabling compacts and flushes for region at 1731093204550Disabling writes for close at 1731093204550Writing region close event to WAL at 1731093204551 (+1 ms)Closed at 1731093204552 (+1 ms) 2024-11-08T19:13:24,555 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T19:13:24,555 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-08T19:13:24,561 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-08T19:13:24,571 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T19:13:24,574 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-08T19:13:24,578 DEBUG [RS:2;692b295ad45c:39647 {}] zookeeper.ZKUtil(111): regionserver:39647-0x1011bddfb000003, quorum=127.0.0.1:56395, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/692b295ad45c,39647,1731093203061 2024-11-08T19:13:24,578 DEBUG [RS:0;692b295ad45c:36007 {}] zookeeper.ZKUtil(111): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/692b295ad45c,36007,1731093202744 2024-11-08T19:13:24,578 WARN [RS:2;692b295ad45c:39647 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T19:13:24,578 WARN [RS:0;692b295ad45c:36007 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T19:13:24,578 INFO [RS:2;692b295ad45c:39647 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T19:13:24,578 INFO [RS:0;692b295ad45c:36007 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T19:13:24,578 DEBUG [RS:2;692b295ad45c:39647 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,39647,1731093203061 2024-11-08T19:13:24,578 DEBUG [RS:0;692b295ad45c:36007 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,36007,1731093202744 2024-11-08T19:13:24,579 DEBUG [RS:1;692b295ad45c:42693 {}] zookeeper.ZKUtil(111): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/692b295ad45c,42693,1731093202977 2024-11-08T19:13:24,579 WARN [RS:1;692b295ad45c:42693 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T19:13:24,579 INFO [RS:1;692b295ad45c:42693 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T19:13:24,580 DEBUG [RS:1;692b295ad45c:42693 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,42693,1731093202977 2024-11-08T19:13:24,581 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [692b295ad45c,39647,1731093203061] 2024-11-08T19:13:24,581 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [692b295ad45c,42693,1731093202977] 2024-11-08T19:13:24,581 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [692b295ad45c,36007,1731093202744] 2024-11-08T19:13:24,607 INFO [RS:2;692b295ad45c:39647 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T19:13:24,607 INFO [RS:1;692b295ad45c:42693 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T19:13:24,607 INFO [RS:0;692b295ad45c:36007 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T19:13:24,623 INFO [RS:1;692b295ad45c:42693 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T19:13:24,623 INFO [RS:2;692b295ad45c:39647 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T19:13:24,624 INFO [RS:0;692b295ad45c:36007 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T19:13:24,630 INFO [RS:1;692b295ad45c:42693 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T19:13:24,630 INFO [RS:2;692b295ad45c:39647 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T19:13:24,630 INFO [RS:0;692b295ad45c:36007 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T19:13:24,631 INFO [RS:1;692b295ad45c:42693 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T19:13:24,631 INFO [RS:2;692b295ad45c:39647 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T19:13:24,631 INFO [RS:0;692b295ad45c:36007 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T19:13:24,636 INFO [RS:0;692b295ad45c:36007 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T19:13:24,636 INFO [RS:1;692b295ad45c:42693 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T19:13:24,636 INFO [RS:2;692b295ad45c:39647 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T19:13:24,642 INFO [RS:2;692b295ad45c:39647 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T19:13:24,642 INFO [RS:1;692b295ad45c:42693 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T19:13:24,644 INFO [RS:1;692b295ad45c:42693 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T19:13:24,644 INFO [RS:2;692b295ad45c:39647 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T19:13:24,644 INFO [RS:0;692b295ad45c:36007 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T19:13:24,644 DEBUG [RS:1;692b295ad45c:42693 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:13:24,644 INFO [RS:0;692b295ad45c:36007 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T19:13:24,644 DEBUG [RS:2;692b295ad45c:39647 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:13:24,644 DEBUG [RS:1;692b295ad45c:42693 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:13:24,644 DEBUG [RS:2;692b295ad45c:39647 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:13:24,644 DEBUG [RS:0;692b295ad45c:36007 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:13:24,645 DEBUG [RS:1;692b295ad45c:42693 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:13:24,645 DEBUG [RS:2;692b295ad45c:39647 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:13:24,645 DEBUG [RS:0;692b295ad45c:36007 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:13:24,645 DEBUG [RS:2;692b295ad45c:39647 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:13:24,645 DEBUG [RS:1;692b295ad45c:42693 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:13:24,645 DEBUG [RS:0;692b295ad45c:36007 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:13:24,645 DEBUG [RS:2;692b295ad45c:39647 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:13:24,645 DEBUG [RS:0;692b295ad45c:36007 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:13:24,645 DEBUG [RS:1;692b295ad45c:42693 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:13:24,645 DEBUG [RS:2;692b295ad45c:39647 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/692b295ad45c:0, corePoolSize=2, maxPoolSize=2 2024-11-08T19:13:24,645 DEBUG [RS:1;692b295ad45c:42693 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/692b295ad45c:0, corePoolSize=2, maxPoolSize=2 2024-11-08T19:13:24,645 DEBUG [RS:2;692b295ad45c:39647 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:13:24,645 DEBUG [RS:1;692b295ad45c:42693 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:13:24,645 DEBUG [RS:2;692b295ad45c:39647 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:13:24,645 DEBUG [RS:1;692b295ad45c:42693 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:13:24,645 DEBUG [RS:2;692b295ad45c:39647 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:13:24,645 DEBUG [RS:1;692b295ad45c:42693 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:13:24,646 DEBUG [RS:2;692b295ad45c:39647 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:13:24,646 DEBUG [RS:1;692b295ad45c:42693 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:13:24,646 DEBUG [RS:2;692b295ad45c:39647 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:13:24,645 DEBUG [RS:0;692b295ad45c:36007 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:13:24,646 DEBUG [RS:1;692b295ad45c:42693 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:13:24,646 DEBUG [RS:2;692b295ad45c:39647 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:13:24,646 DEBUG [RS:0;692b295ad45c:36007 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/692b295ad45c:0, corePoolSize=2, maxPoolSize=2 2024-11-08T19:13:24,646 DEBUG [RS:1;692b295ad45c:42693 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:13:24,646 DEBUG [RS:2;692b295ad45c:39647 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/692b295ad45c:0, corePoolSize=3, maxPoolSize=3 2024-11-08T19:13:24,646 DEBUG [RS:1;692b295ad45c:42693 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/692b295ad45c:0, corePoolSize=3, maxPoolSize=3 2024-11-08T19:13:24,646 DEBUG [RS:2;692b295ad45c:39647 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/692b295ad45c:0, corePoolSize=3, maxPoolSize=3 2024-11-08T19:13:24,646 DEBUG [RS:1;692b295ad45c:42693 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/692b295ad45c:0, corePoolSize=3, maxPoolSize=3 2024-11-08T19:13:24,646 DEBUG [RS:0;692b295ad45c:36007 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:13:24,647 DEBUG [RS:0;692b295ad45c:36007 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:13:24,647 DEBUG [RS:0;692b295ad45c:36007 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:13:24,647 DEBUG [RS:0;692b295ad45c:36007 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:13:24,647 DEBUG [RS:0;692b295ad45c:36007 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:13:24,647 DEBUG [RS:0;692b295ad45c:36007 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:13:24,647 DEBUG [RS:0;692b295ad45c:36007 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/692b295ad45c:0, corePoolSize=3, maxPoolSize=3 2024-11-08T19:13:24,647 DEBUG [RS:0;692b295ad45c:36007 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/692b295ad45c:0, corePoolSize=3, maxPoolSize=3 2024-11-08T19:13:24,653 INFO [RS:2;692b295ad45c:39647 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T19:13:24,653 INFO [RS:2;692b295ad45c:39647 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T19:13:24,653 INFO [RS:2;692b295ad45c:39647 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T19:13:24,653 INFO [RS:2;692b295ad45c:39647 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T19:13:24,653 INFO [RS:1;692b295ad45c:42693 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T19:13:24,653 INFO [RS:2;692b295ad45c:39647 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T19:13:24,654 INFO [RS:0;692b295ad45c:36007 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T19:13:24,654 INFO [RS:2;692b295ad45c:39647 {}] hbase.ChoreService(168): Chore ScheduledChore name=692b295ad45c,39647,1731093203061-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T19:13:24,654 INFO [RS:1;692b295ad45c:42693 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T19:13:24,654 INFO [RS:0;692b295ad45c:36007 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T19:13:24,654 INFO [RS:1;692b295ad45c:42693 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T19:13:24,654 INFO [RS:0;692b295ad45c:36007 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T19:13:24,654 INFO [RS:1;692b295ad45c:42693 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T19:13:24,654 INFO [RS:0;692b295ad45c:36007 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T19:13:24,654 INFO [RS:1;692b295ad45c:42693 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T19:13:24,654 INFO [RS:0;692b295ad45c:36007 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T19:13:24,654 INFO [RS:1;692b295ad45c:42693 {}] hbase.ChoreService(168): Chore ScheduledChore name=692b295ad45c,42693,1731093202977-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T19:13:24,654 INFO [RS:0;692b295ad45c:36007 {}] hbase.ChoreService(168): Chore ScheduledChore name=692b295ad45c,36007,1731093202744-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T19:13:24,672 INFO [RS:2;692b295ad45c:39647 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T19:13:24,674 INFO [RS:2;692b295ad45c:39647 {}] hbase.ChoreService(168): Chore ScheduledChore name=692b295ad45c,39647,1731093203061-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T19:13:24,675 INFO [RS:2;692b295ad45c:39647 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T19:13:24,675 INFO [RS:0;692b295ad45c:36007 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T19:13:24,675 INFO [RS:2;692b295ad45c:39647 {}] regionserver.Replication(171): 692b295ad45c,39647,1731093203061 started 2024-11-08T19:13:24,675 INFO [RS:0;692b295ad45c:36007 {}] hbase.ChoreService(168): Chore ScheduledChore name=692b295ad45c,36007,1731093202744-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T19:13:24,675 INFO [RS:0;692b295ad45c:36007 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T19:13:24,675 INFO [RS:0;692b295ad45c:36007 {}] regionserver.Replication(171): 692b295ad45c,36007,1731093202744 started 2024-11-08T19:13:24,680 INFO [RS:1;692b295ad45c:42693 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T19:13:24,680 INFO [RS:1;692b295ad45c:42693 {}] hbase.ChoreService(168): Chore ScheduledChore name=692b295ad45c,42693,1731093202977-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T19:13:24,680 INFO [RS:1;692b295ad45c:42693 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T19:13:24,681 INFO [RS:1;692b295ad45c:42693 {}] regionserver.Replication(171): 692b295ad45c,42693,1731093202977 started 2024-11-08T19:13:24,694 INFO [RS:2;692b295ad45c:39647 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T19:13:24,694 INFO [RS:0;692b295ad45c:36007 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T19:13:24,694 INFO [RS:0;692b295ad45c:36007 {}] regionserver.HRegionServer(1482): Serving as 692b295ad45c,36007,1731093202744, RpcServer on 692b295ad45c/172.17.0.2:36007, sessionid=0x1011bddfb000001 2024-11-08T19:13:24,694 INFO [RS:2;692b295ad45c:39647 {}] regionserver.HRegionServer(1482): Serving as 692b295ad45c,39647,1731093203061, RpcServer on 692b295ad45c/172.17.0.2:39647, sessionid=0x1011bddfb000003 2024-11-08T19:13:24,695 DEBUG [RS:0;692b295ad45c:36007 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T19:13:24,695 DEBUG [RS:2;692b295ad45c:39647 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T19:13:24,695 DEBUG [RS:2;692b295ad45c:39647 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 692b295ad45c,39647,1731093203061 2024-11-08T19:13:24,695 DEBUG [RS:0;692b295ad45c:36007 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 692b295ad45c,36007,1731093202744 2024-11-08T19:13:24,695 DEBUG [RS:2;692b295ad45c:39647 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '692b295ad45c,39647,1731093203061' 2024-11-08T19:13:24,695 DEBUG [RS:0;692b295ad45c:36007 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '692b295ad45c,36007,1731093202744' 2024-11-08T19:13:24,695 DEBUG [RS:2;692b295ad45c:39647 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T19:13:24,695 DEBUG [RS:0;692b295ad45c:36007 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T19:13:24,696 DEBUG [RS:0;692b295ad45c:36007 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T19:13:24,696 DEBUG [RS:2;692b295ad45c:39647 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T19:13:24,696 DEBUG [RS:2;692b295ad45c:39647 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T19:13:24,696 DEBUG [RS:2;692b295ad45c:39647 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T19:13:24,697 DEBUG [RS:0;692b295ad45c:36007 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T19:13:24,697 DEBUG [RS:0;692b295ad45c:36007 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T19:13:24,697 DEBUG [RS:2;692b295ad45c:39647 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 692b295ad45c,39647,1731093203061 2024-11-08T19:13:24,697 DEBUG [RS:0;692b295ad45c:36007 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 692b295ad45c,36007,1731093202744 2024-11-08T19:13:24,697 DEBUG [RS:2;692b295ad45c:39647 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '692b295ad45c,39647,1731093203061' 2024-11-08T19:13:24,697 DEBUG [RS:2;692b295ad45c:39647 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T19:13:24,697 DEBUG [RS:0;692b295ad45c:36007 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '692b295ad45c,36007,1731093202744' 2024-11-08T19:13:24,697 DEBUG [RS:0;692b295ad45c:36007 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T19:13:24,697 DEBUG [RS:2;692b295ad45c:39647 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T19:13:24,697 DEBUG [RS:0;692b295ad45c:36007 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T19:13:24,698 DEBUG [RS:2;692b295ad45c:39647 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T19:13:24,698 DEBUG [RS:0;692b295ad45c:36007 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T19:13:24,698 INFO [RS:2;692b295ad45c:39647 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T19:13:24,698 INFO [RS:0;692b295ad45c:36007 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T19:13:24,698 INFO [RS:2;692b295ad45c:39647 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T19:13:24,698 INFO [RS:0;692b295ad45c:36007 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T19:13:24,703 INFO [RS:1;692b295ad45c:42693 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T19:13:24,703 INFO [RS:1;692b295ad45c:42693 {}] regionserver.HRegionServer(1482): Serving as 692b295ad45c,42693,1731093202977, RpcServer on 692b295ad45c/172.17.0.2:42693, sessionid=0x1011bddfb000002 2024-11-08T19:13:24,703 DEBUG [RS:1;692b295ad45c:42693 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T19:13:24,703 DEBUG [RS:1;692b295ad45c:42693 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 692b295ad45c,42693,1731093202977 2024-11-08T19:13:24,703 DEBUG [RS:1;692b295ad45c:42693 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '692b295ad45c,42693,1731093202977' 2024-11-08T19:13:24,703 DEBUG [RS:1;692b295ad45c:42693 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T19:13:24,704 DEBUG [RS:1;692b295ad45c:42693 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T19:13:24,705 DEBUG [RS:1;692b295ad45c:42693 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T19:13:24,705 DEBUG [RS:1;692b295ad45c:42693 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T19:13:24,705 DEBUG [RS:1;692b295ad45c:42693 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 692b295ad45c,42693,1731093202977 2024-11-08T19:13:24,705 DEBUG [RS:1;692b295ad45c:42693 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '692b295ad45c,42693,1731093202977' 2024-11-08T19:13:24,705 DEBUG [RS:1;692b295ad45c:42693 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T19:13:24,706 DEBUG [RS:1;692b295ad45c:42693 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T19:13:24,707 DEBUG [RS:1;692b295ad45c:42693 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T19:13:24,707 INFO [RS:1;692b295ad45c:42693 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T19:13:24,707 INFO [RS:1;692b295ad45c:42693 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T19:13:24,726 WARN [692b295ad45c:39509 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-08T19:13:24,804 INFO [RS:2;692b295ad45c:39647 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T19:13:24,804 INFO [RS:0;692b295ad45c:36007 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T19:13:24,808 INFO [RS:1;692b295ad45c:42693 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T19:13:24,808 INFO [RS:0;692b295ad45c:36007 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=692b295ad45c%2C36007%2C1731093202744, suffix=, logDir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,36007,1731093202744, archiveDir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/oldWALs, maxLogs=32 2024-11-08T19:13:24,808 INFO [RS:2;692b295ad45c:39647 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=692b295ad45c%2C39647%2C1731093203061, suffix=, logDir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,39647,1731093203061, archiveDir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/oldWALs, maxLogs=32 2024-11-08T19:13:24,810 INFO [RS:1;692b295ad45c:42693 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=692b295ad45c%2C42693%2C1731093202977, suffix=, logDir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,42693,1731093202977, archiveDir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/oldWALs, maxLogs=32 2024-11-08T19:13:24,825 DEBUG [RS:1;692b295ad45c:42693 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,42693,1731093202977/692b295ad45c%2C42693%2C1731093202977.1731093204812, exclude list is [], retry=0 2024-11-08T19:13:24,827 DEBUG [RS:2;692b295ad45c:39647 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,39647,1731093203061/692b295ad45c%2C39647%2C1731093203061.1731093204811, exclude list is [], retry=0 2024-11-08T19:13:24,829 DEBUG [RS:0;692b295ad45c:36007 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,36007,1731093202744/692b295ad45c%2C36007%2C1731093202744.1731093204811, exclude list is [], retry=0 2024-11-08T19:13:24,831 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46657,DS-5e1bf434-3af5-4fc9-abe7-9a902de021b6,DISK] 2024-11-08T19:13:24,832 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40347,DS-6e01471a-b8b4-43a1-88b3-b57bf9ad1710,DISK] 2024-11-08T19:13:24,832 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42249,DS-c0e88d76-cfe1-4c35-bbea-3e8b1423e597,DISK] 2024-11-08T19:13:24,833 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46657,DS-5e1bf434-3af5-4fc9-abe7-9a902de021b6,DISK] 2024-11-08T19:13:24,833 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40347,DS-6e01471a-b8b4-43a1-88b3-b57bf9ad1710,DISK] 2024-11-08T19:13:24,834 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42249,DS-c0e88d76-cfe1-4c35-bbea-3e8b1423e597,DISK] 2024-11-08T19:13:24,835 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46657,DS-5e1bf434-3af5-4fc9-abe7-9a902de021b6,DISK] 2024-11-08T19:13:24,835 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42249,DS-c0e88d76-cfe1-4c35-bbea-3e8b1423e597,DISK] 2024-11-08T19:13:24,835 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40347,DS-6e01471a-b8b4-43a1-88b3-b57bf9ad1710,DISK] 2024-11-08T19:13:24,850 INFO [RS:1;692b295ad45c:42693 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,42693,1731093202977/692b295ad45c%2C42693%2C1731093202977.1731093204812 2024-11-08T19:13:24,850 INFO [RS:2;692b295ad45c:39647 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,39647,1731093203061/692b295ad45c%2C39647%2C1731093203061.1731093204811 2024-11-08T19:13:24,851 DEBUG [RS:1;692b295ad45c:42693 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36293:36293),(127.0.0.1/127.0.0.1:39837:39837),(127.0.0.1/127.0.0.1:46659:46659)] 2024-11-08T19:13:24,851 DEBUG [RS:2;692b295ad45c:39647 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46659:46659),(127.0.0.1/127.0.0.1:39837:39837),(127.0.0.1/127.0.0.1:36293:36293)] 2024-11-08T19:13:24,852 INFO [RS:0;692b295ad45c:36007 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,36007,1731093202744/692b295ad45c%2C36007%2C1731093202744.1731093204811 2024-11-08T19:13:24,855 DEBUG [RS:0;692b295ad45c:36007 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46659:46659),(127.0.0.1/127.0.0.1:36293:36293),(127.0.0.1/127.0.0.1:39837:39837)] 2024-11-08T19:13:24,978 DEBUG [692b295ad45c:39509 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-08T19:13:24,986 DEBUG [692b295ad45c:39509 {}] balancer.BalancerClusterState(204): Hosts are {692b295ad45c=0} racks are {/default-rack=0} 2024-11-08T19:13:24,992 DEBUG [692b295ad45c:39509 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T19:13:24,992 DEBUG [692b295ad45c:39509 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T19:13:24,992 DEBUG [692b295ad45c:39509 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-08T19:13:24,992 DEBUG [692b295ad45c:39509 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T19:13:24,992 DEBUG [692b295ad45c:39509 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T19:13:24,992 DEBUG [692b295ad45c:39509 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-08T19:13:24,992 INFO [692b295ad45c:39509 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T19:13:24,992 INFO [692b295ad45c:39509 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T19:13:24,992 INFO [692b295ad45c:39509 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-08T19:13:24,992 DEBUG [692b295ad45c:39509 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T19:13:25,000 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=692b295ad45c,39647,1731093203061 2024-11-08T19:13:25,006 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 692b295ad45c,39647,1731093203061, state=OPENING 2024-11-08T19:13:25,117 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-08T19:13:25,128 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T19:13:25,128 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39647-0x1011bddfb000003, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T19:13:25,128 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T19:13:25,128 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T19:13:25,130 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:25,131 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:25,131 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:25,131 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:25,133 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T19:13:25,135 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,39647,1731093203061}] 2024-11-08T19:13:25,314 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T19:13:25,316 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59579, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T19:13:25,327 INFO [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-08T19:13:25,328 INFO [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T19:13:25,328 INFO [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-08T19:13:25,331 INFO [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=692b295ad45c%2C39647%2C1731093203061.meta, suffix=.meta, logDir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,39647,1731093203061, archiveDir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/oldWALs, maxLogs=32 2024-11-08T19:13:25,349 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,39647,1731093203061/692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, exclude list is [], retry=0 2024-11-08T19:13:25,353 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42249,DS-c0e88d76-cfe1-4c35-bbea-3e8b1423e597,DISK] 2024-11-08T19:13:25,353 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46657,DS-5e1bf434-3af5-4fc9-abe7-9a902de021b6,DISK] 2024-11-08T19:13:25,353 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40347,DS-6e01471a-b8b4-43a1-88b3-b57bf9ad1710,DISK] 2024-11-08T19:13:25,356 INFO [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,39647,1731093203061/692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta 2024-11-08T19:13:25,356 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46659:46659),(127.0.0.1/127.0.0.1:36293:36293),(127.0.0.1/127.0.0.1:39837:39837)] 2024-11-08T19:13:25,356 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-08T19:13:25,358 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-08T19:13:25,360 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-08T19:13:25,365 INFO [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-08T19:13:25,368 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-08T19:13:25,369 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:25,369 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-08T19:13:25,369 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-08T19:13:25,372 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-08T19:13:25,373 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T19:13:25,373 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:25,374 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T19:13:25,374 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-08T19:13:25,376 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T19:13:25,376 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:25,377 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T19:13:25,377 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-08T19:13:25,379 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T19:13:25,379 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:25,380 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T19:13:25,380 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-08T19:13:25,381 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T19:13:25,381 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:25,382 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T19:13:25,382 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T19:13:25,384 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/hbase/meta/1588230740 2024-11-08T19:13:25,387 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/hbase/meta/1588230740 2024-11-08T19:13:25,390 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T19:13:25,390 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T19:13:25,391 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T19:13:25,394 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T19:13:25,396 INFO [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61994035, jitterRate=-0.07621689140796661}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T19:13:25,396 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-08T19:13:25,398 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731093205369Writing region info on filesystem at 1731093205370 (+1 ms)Initializing all the Stores at 1731093205371 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731093205371Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731093205372 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731093205372Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731093205372Cleaning up temporary data from old regions at 1731093205390 (+18 ms)Running coprocessor post-open hooks at 1731093205396 (+6 ms)Region opened successfully at 1731093205398 (+2 ms) 2024-11-08T19:13:25,406 INFO [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731093205304 2024-11-08T19:13:25,418 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-08T19:13:25,419 INFO [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-08T19:13:25,420 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=692b295ad45c,39647,1731093203061 2024-11-08T19:13:25,423 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 692b295ad45c,39647,1731093203061, state=OPEN 2024-11-08T19:13:25,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:25,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:25,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39647-0x1011bddfb000003, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:25,432 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:25,432 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:25,432 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:25,432 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:25,432 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:25,433 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=692b295ad45c,39647,1731093203061 2024-11-08T19:13:25,440 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-08T19:13:25,440 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,39647,1731093203061 in 298 msec 2024-11-08T19:13:25,448 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-08T19:13:25,448 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 883 msec 2024-11-08T19:13:25,450 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T19:13:25,450 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-08T19:13:25,469 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T19:13:25,470 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=692b295ad45c,39647,1731093203061, seqNum=-1] 2024-11-08T19:13:25,489 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T19:13:25,492 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38705, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T19:13:25,512 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1970 sec 2024-11-08T19:13:25,512 INFO [master/692b295ad45c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731093205512, completionTime=-1 2024-11-08T19:13:25,515 INFO [master/692b295ad45c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-08T19:13:25,516 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T19:13:25,545 INFO [master/692b295ad45c:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=3 2024-11-08T19:13:25,545 INFO [master/692b295ad45c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731093265545 2024-11-08T19:13:25,545 INFO [master/692b295ad45c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731093325545 2024-11-08T19:13:25,545 INFO [master/692b295ad45c:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 29 msec 2024-11-08T19:13:25,546 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-08T19:13:25,553 INFO [master/692b295ad45c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=692b295ad45c,39509,1731093201567-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T19:13:25,553 INFO [master/692b295ad45c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=692b295ad45c,39509,1731093201567-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T19:13:25,553 INFO [master/692b295ad45c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=692b295ad45c,39509,1731093201567-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T19:13:25,555 INFO [master/692b295ad45c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-692b295ad45c:39509, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T19:13:25,555 INFO [master/692b295ad45c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-08T19:13:25,556 INFO [master/692b295ad45c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-08T19:13:25,562 DEBUG [master/692b295ad45c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-08T19:13:25,586 INFO [master/692b295ad45c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.262sec 2024-11-08T19:13:25,587 INFO [master/692b295ad45c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-08T19:13:25,588 INFO [master/692b295ad45c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-08T19:13:25,589 INFO [master/692b295ad45c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-08T19:13:25,590 INFO [master/692b295ad45c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-08T19:13:25,590 INFO [master/692b295ad45c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-08T19:13:25,591 INFO [master/692b295ad45c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=692b295ad45c,39509,1731093201567-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T19:13:25,591 INFO [master/692b295ad45c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=692b295ad45c,39509,1731093201567-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-08T19:13:25,597 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-08T19:13:25,598 INFO [master/692b295ad45c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-08T19:13:25,598 INFO [master/692b295ad45c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=692b295ad45c,39509,1731093201567-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T19:13:25,638 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39f0836e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T19:13:25,641 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-08T19:13:25,641 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-08T19:13:25,644 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 692b295ad45c,39509,-1 for getting cluster id 2024-11-08T19:13:25,646 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T19:13:25,654 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c2739604-5628-4fa2-a3e8-33881b328b44' 2024-11-08T19:13:25,656 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T19:13:25,656 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c2739604-5628-4fa2-a3e8-33881b328b44" 2024-11-08T19:13:25,657 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2709e261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T19:13:25,657 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [692b295ad45c,39509,-1] 2024-11-08T19:13:25,659 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T19:13:25,661 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T19:13:25,663 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T19:13:25,666 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41704ed8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T19:13:25,666 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T19:13:25,673 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=692b295ad45c,39647,1731093203061, seqNum=-1] 2024-11-08T19:13:25,674 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T19:13:25,676 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34192, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T19:13:25,695 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=692b295ad45c,39509,1731093201567 2024-11-08T19:13:25,698 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T19:13:25,722 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 692b295ad45c,39509,1731093201567 2024-11-08T19:13:25,725 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@37e38f6b 2024-11-08T19:13:25,726 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T19:13:25,729 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42884, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T19:13:25,735 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T19:13:25,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-11-08T19:13:25,750 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T19:13:25,753 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-11-08T19:13:25,754 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:25,757 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T19:13:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T19:13:25,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741837_1013 (size=358) 2024-11-08T19:13:25,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741837_1013 (size=358) 2024-11-08T19:13:25,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741837_1013 (size=358) 2024-11-08T19:13:25,779 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d2ea8472f395c484d4998844c4dcd304, NAME => 'test,,1731093205731.d2ea8472f395c484d4998844c4dcd304.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f 2024-11-08T19:13:25,780 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 7b4a23d5efe8e832c0d5cb16df3b1369, NAME => 'test,ccc,1731093205731.7b4a23d5efe8e832c0d5cb16df3b1369.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f 2024-11-08T19:13:25,780 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 8cb2ac5239e07e076f513604c29c2ca8, NAME => 'test,bbb,1731093205731.8cb2ac5239e07e076f513604c29c2ca8.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f 2024-11-08T19:13:25,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741838_1014 (size=45) 2024-11-08T19:13:25,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741838_1014 (size=45) 2024-11-08T19:13:25,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741838_1014 (size=45) 2024-11-08T19:13:25,800 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1731093205731.7b4a23d5efe8e832c0d5cb16df3b1369.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:25,800 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 7b4a23d5efe8e832c0d5cb16df3b1369, disabling compactions & flushes 2024-11-08T19:13:25,801 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1731093205731.7b4a23d5efe8e832c0d5cb16df3b1369. 2024-11-08T19:13:25,801 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1731093205731.7b4a23d5efe8e832c0d5cb16df3b1369. 2024-11-08T19:13:25,801 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1731093205731.7b4a23d5efe8e832c0d5cb16df3b1369. after waiting 0 ms 2024-11-08T19:13:25,801 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1731093205731.7b4a23d5efe8e832c0d5cb16df3b1369. 2024-11-08T19:13:25,801 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1731093205731.7b4a23d5efe8e832c0d5cb16df3b1369. 2024-11-08T19:13:25,801 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 7b4a23d5efe8e832c0d5cb16df3b1369: Waiting for close lock at 1731093205800Disabling compacts and flushes for region at 1731093205800Disabling writes for close at 1731093205801 (+1 ms)Writing region close event to WAL at 1731093205801Closed at 1731093205801 2024-11-08T19:13:25,802 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => a8e891c0073a08913834761f455a77e8, NAME => 'test,ddd,1731093205731.a8e891c0073a08913834761f455a77e8.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f 2024-11-08T19:13:25,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741839_1015 (size=42) 2024-11-08T19:13:25,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741839_1015 (size=42) 2024-11-08T19:13:25,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741839_1015 (size=42) 2024-11-08T19:13:25,806 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1731093205731.d2ea8472f395c484d4998844c4dcd304.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:25,806 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing d2ea8472f395c484d4998844c4dcd304, disabling compactions & flushes 2024-11-08T19:13:25,806 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1731093205731.d2ea8472f395c484d4998844c4dcd304. 2024-11-08T19:13:25,806 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1731093205731.d2ea8472f395c484d4998844c4dcd304. 2024-11-08T19:13:25,806 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1731093205731.d2ea8472f395c484d4998844c4dcd304. after waiting 0 ms 2024-11-08T19:13:25,806 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1731093205731.d2ea8472f395c484d4998844c4dcd304. 2024-11-08T19:13:25,806 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1731093205731.d2ea8472f395c484d4998844c4dcd304. 2024-11-08T19:13:25,806 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for d2ea8472f395c484d4998844c4dcd304: Waiting for close lock at 1731093205806Disabling compacts and flushes for region at 1731093205806Disabling writes for close at 1731093205806Writing region close event to WAL at 1731093205806Closed at 1731093205806 2024-11-08T19:13:25,807 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fdc84e1c2d9bdd5f39d647a57ff91a05, NAME => 'test,eee,1731093205731.fdc84e1c2d9bdd5f39d647a57ff91a05.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f 2024-11-08T19:13:25,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741840_1016 (size=45) 2024-11-08T19:13:25,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741840_1016 (size=45) 2024-11-08T19:13:25,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741840_1016 (size=45) 2024-11-08T19:13:25,820 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1731093205731.8cb2ac5239e07e076f513604c29c2ca8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:25,820 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 8cb2ac5239e07e076f513604c29c2ca8, disabling compactions & flushes 2024-11-08T19:13:25,820 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1731093205731.8cb2ac5239e07e076f513604c29c2ca8. 2024-11-08T19:13:25,820 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1731093205731.8cb2ac5239e07e076f513604c29c2ca8. 2024-11-08T19:13:25,820 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1731093205731.8cb2ac5239e07e076f513604c29c2ca8. after waiting 0 ms 2024-11-08T19:13:25,820 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1731093205731.8cb2ac5239e07e076f513604c29c2ca8. 2024-11-08T19:13:25,820 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1731093205731.8cb2ac5239e07e076f513604c29c2ca8. 2024-11-08T19:13:25,820 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 8cb2ac5239e07e076f513604c29c2ca8: Waiting for close lock at 1731093205820Disabling compacts and flushes for region at 1731093205820Disabling writes for close at 1731093205820Writing region close event to WAL at 1731093205820Closed at 1731093205820 2024-11-08T19:13:25,821 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 53b7061bfce2939f050c57221eed7f2a, NAME => 'test,fff,1731093205731.53b7061bfce2939f050c57221eed7f2a.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f 2024-11-08T19:13:25,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741841_1017 (size=45) 2024-11-08T19:13:25,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741841_1017 (size=45) 2024-11-08T19:13:25,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741841_1017 (size=45) 2024-11-08T19:13:25,834 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ddd,1731093205731.a8e891c0073a08913834761f455a77e8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:25,834 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing a8e891c0073a08913834761f455a77e8, disabling compactions & flushes 2024-11-08T19:13:25,834 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ddd,1731093205731.a8e891c0073a08913834761f455a77e8. 2024-11-08T19:13:25,834 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1731093205731.a8e891c0073a08913834761f455a77e8. 2024-11-08T19:13:25,834 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1731093205731.a8e891c0073a08913834761f455a77e8. after waiting 0 ms 2024-11-08T19:13:25,834 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1731093205731.a8e891c0073a08913834761f455a77e8. 2024-11-08T19:13:25,834 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ddd,1731093205731.a8e891c0073a08913834761f455a77e8. 2024-11-08T19:13:25,834 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for a8e891c0073a08913834761f455a77e8: Waiting for close lock at 1731093205834Disabling compacts and flushes for region at 1731093205834Disabling writes for close at 1731093205834Writing region close event to WAL at 1731093205834Closed at 1731093205834 2024-11-08T19:13:25,835 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => ab92690881683c86166f7f23976b5188, NAME => 'test,ggg,1731093205731.ab92690881683c86166f7f23976b5188.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f 2024-11-08T19:13:25,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741842_1018 (size=45) 2024-11-08T19:13:25,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741842_1018 (size=45) 2024-11-08T19:13:25,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741842_1018 (size=45) 2024-11-08T19:13:25,839 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,eee,1731093205731.fdc84e1c2d9bdd5f39d647a57ff91a05.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:25,840 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing fdc84e1c2d9bdd5f39d647a57ff91a05, disabling compactions & flushes 2024-11-08T19:13:25,840 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,eee,1731093205731.fdc84e1c2d9bdd5f39d647a57ff91a05. 2024-11-08T19:13:25,840 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1731093205731.fdc84e1c2d9bdd5f39d647a57ff91a05. 2024-11-08T19:13:25,840 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1731093205731.fdc84e1c2d9bdd5f39d647a57ff91a05. after waiting 0 ms 2024-11-08T19:13:25,840 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1731093205731.fdc84e1c2d9bdd5f39d647a57ff91a05. 2024-11-08T19:13:25,840 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,eee,1731093205731.fdc84e1c2d9bdd5f39d647a57ff91a05. 2024-11-08T19:13:25,840 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for fdc84e1c2d9bdd5f39d647a57ff91a05: Waiting for close lock at 1731093205840Disabling compacts and flushes for region at 1731093205840Disabling writes for close at 1731093205840Writing region close event to WAL at 1731093205840Closed at 1731093205840 2024-11-08T19:13:25,841 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7d1d9fe849fd45b830c330834b58d508, NAME => 'test,hhh,1731093205731.7d1d9fe849fd45b830c330834b58d508.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f 2024-11-08T19:13:25,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741843_1019 (size=45) 2024-11-08T19:13:25,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741843_1019 (size=45) 2024-11-08T19:13:25,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741843_1019 (size=45) 2024-11-08T19:13:25,850 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,fff,1731093205731.53b7061bfce2939f050c57221eed7f2a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:25,850 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 53b7061bfce2939f050c57221eed7f2a, disabling compactions & flushes 2024-11-08T19:13:25,850 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,fff,1731093205731.53b7061bfce2939f050c57221eed7f2a. 2024-11-08T19:13:25,850 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1731093205731.53b7061bfce2939f050c57221eed7f2a. 2024-11-08T19:13:25,850 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1731093205731.53b7061bfce2939f050c57221eed7f2a. after waiting 0 ms 2024-11-08T19:13:25,850 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1731093205731.53b7061bfce2939f050c57221eed7f2a. 2024-11-08T19:13:25,850 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,fff,1731093205731.53b7061bfce2939f050c57221eed7f2a. 2024-11-08T19:13:25,850 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 53b7061bfce2939f050c57221eed7f2a: Waiting for close lock at 1731093205850Disabling compacts and flushes for region at 1731093205850Disabling writes for close at 1731093205850Writing region close event to WAL at 1731093205850Closed at 1731093205850 2024-11-08T19:13:25,851 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 2548b6c044cd2c9ceb7dec340e2cde6c, NAME => 'test,iii,1731093205731.2548b6c044cd2c9ceb7dec340e2cde6c.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f 2024-11-08T19:13:25,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741844_1020 (size=45) 2024-11-08T19:13:25,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741844_1020 (size=45) 2024-11-08T19:13:25,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741844_1020 (size=45) 2024-11-08T19:13:25,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T19:13:25,872 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ggg,1731093205731.ab92690881683c86166f7f23976b5188.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:25,872 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing ab92690881683c86166f7f23976b5188, disabling compactions & flushes 2024-11-08T19:13:25,873 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ggg,1731093205731.ab92690881683c86166f7f23976b5188. 2024-11-08T19:13:25,873 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1731093205731.ab92690881683c86166f7f23976b5188. 2024-11-08T19:13:25,873 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1731093205731.ab92690881683c86166f7f23976b5188. after waiting 0 ms 2024-11-08T19:13:25,873 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1731093205731.ab92690881683c86166f7f23976b5188. 2024-11-08T19:13:25,873 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ggg,1731093205731.ab92690881683c86166f7f23976b5188. 2024-11-08T19:13:25,873 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for ab92690881683c86166f7f23976b5188: Waiting for close lock at 1731093205872Disabling compacts and flushes for region at 1731093205872Disabling writes for close at 1731093205873 (+1 ms)Writing region close event to WAL at 1731093205873Closed at 1731093205873 2024-11-08T19:13:25,873 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => f7a8c4f9e9448f30c950ddcf996ae56f, NAME => 'test,jjj,1731093205731.f7a8c4f9e9448f30c950ddcf996ae56f.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f 2024-11-08T19:13:25,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741845_1021 (size=45) 2024-11-08T19:13:25,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741845_1021 (size=45) 2024-11-08T19:13:25,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741845_1021 (size=45) 2024-11-08T19:13:25,879 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,hhh,1731093205731.7d1d9fe849fd45b830c330834b58d508.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:25,879 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 7d1d9fe849fd45b830c330834b58d508, disabling compactions & flushes 2024-11-08T19:13:25,879 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,hhh,1731093205731.7d1d9fe849fd45b830c330834b58d508. 2024-11-08T19:13:25,879 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1731093205731.7d1d9fe849fd45b830c330834b58d508. 2024-11-08T19:13:25,879 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1731093205731.7d1d9fe849fd45b830c330834b58d508. after waiting 0 ms 2024-11-08T19:13:25,879 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1731093205731.7d1d9fe849fd45b830c330834b58d508. 2024-11-08T19:13:25,879 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,hhh,1731093205731.7d1d9fe849fd45b830c330834b58d508. 2024-11-08T19:13:25,879 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7d1d9fe849fd45b830c330834b58d508: Waiting for close lock at 1731093205879Disabling compacts and flushes for region at 1731093205879Disabling writes for close at 1731093205879Writing region close event to WAL at 1731093205879Closed at 1731093205879 2024-11-08T19:13:25,880 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3d0557ff953666f2f97a7be0e2d0bc87, NAME => 'test,kkk,1731093205731.3d0557ff953666f2f97a7be0e2d0bc87.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f 2024-11-08T19:13:25,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741846_1022 (size=45) 2024-11-08T19:13:25,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741846_1022 (size=45) 2024-11-08T19:13:25,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741846_1022 (size=45) 2024-11-08T19:13:25,887 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,iii,1731093205731.2548b6c044cd2c9ceb7dec340e2cde6c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:25,887 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 2548b6c044cd2c9ceb7dec340e2cde6c, disabling compactions & flushes 2024-11-08T19:13:25,887 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,iii,1731093205731.2548b6c044cd2c9ceb7dec340e2cde6c. 2024-11-08T19:13:25,887 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1731093205731.2548b6c044cd2c9ceb7dec340e2cde6c. 2024-11-08T19:13:25,887 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1731093205731.2548b6c044cd2c9ceb7dec340e2cde6c. after waiting 0 ms 2024-11-08T19:13:25,887 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1731093205731.2548b6c044cd2c9ceb7dec340e2cde6c. 2024-11-08T19:13:25,887 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,iii,1731093205731.2548b6c044cd2c9ceb7dec340e2cde6c. 2024-11-08T19:13:25,887 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 2548b6c044cd2c9ceb7dec340e2cde6c: Waiting for close lock at 1731093205887Disabling compacts and flushes for region at 1731093205887Disabling writes for close at 1731093205887Writing region close event to WAL at 1731093205887Closed at 1731093205887 2024-11-08T19:13:25,888 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 6e461a86c03ca80ab184602fce3cc56c, NAME => 'test,lll,1731093205731.6e461a86c03ca80ab184602fce3cc56c.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f 2024-11-08T19:13:25,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741847_1023 (size=45) 2024-11-08T19:13:25,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741847_1023 (size=45) 2024-11-08T19:13:25,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741847_1023 (size=45) 2024-11-08T19:13:25,893 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,jjj,1731093205731.f7a8c4f9e9448f30c950ddcf996ae56f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:25,894 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing f7a8c4f9e9448f30c950ddcf996ae56f, disabling compactions & flushes 2024-11-08T19:13:25,894 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,jjj,1731093205731.f7a8c4f9e9448f30c950ddcf996ae56f. 2024-11-08T19:13:25,894 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1731093205731.f7a8c4f9e9448f30c950ddcf996ae56f. 2024-11-08T19:13:25,894 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1731093205731.f7a8c4f9e9448f30c950ddcf996ae56f. after waiting 0 ms 2024-11-08T19:13:25,894 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1731093205731.f7a8c4f9e9448f30c950ddcf996ae56f. 2024-11-08T19:13:25,894 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,jjj,1731093205731.f7a8c4f9e9448f30c950ddcf996ae56f. 2024-11-08T19:13:25,894 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for f7a8c4f9e9448f30c950ddcf996ae56f: Waiting for close lock at 1731093205894Disabling compacts and flushes for region at 1731093205894Disabling writes for close at 1731093205894Writing region close event to WAL at 1731093205894Closed at 1731093205894 2024-11-08T19:13:25,894 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 245339f649708a4ea812249dd3f822c4, NAME => 'test,mmm,1731093205731.245339f649708a4ea812249dd3f822c4.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f 2024-11-08T19:13:25,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741848_1024 (size=45) 2024-11-08T19:13:25,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741848_1024 (size=45) 2024-11-08T19:13:25,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741848_1024 (size=45) 2024-11-08T19:13:25,903 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,kkk,1731093205731.3d0557ff953666f2f97a7be0e2d0bc87.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:25,903 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 3d0557ff953666f2f97a7be0e2d0bc87, disabling compactions & flushes 2024-11-08T19:13:25,903 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,kkk,1731093205731.3d0557ff953666f2f97a7be0e2d0bc87. 2024-11-08T19:13:25,903 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1731093205731.3d0557ff953666f2f97a7be0e2d0bc87. 2024-11-08T19:13:25,903 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1731093205731.3d0557ff953666f2f97a7be0e2d0bc87. after waiting 0 ms 2024-11-08T19:13:25,903 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1731093205731.3d0557ff953666f2f97a7be0e2d0bc87. 2024-11-08T19:13:25,903 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,kkk,1731093205731.3d0557ff953666f2f97a7be0e2d0bc87. 2024-11-08T19:13:25,904 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3d0557ff953666f2f97a7be0e2d0bc87: Waiting for close lock at 1731093205903Disabling compacts and flushes for region at 1731093205903Disabling writes for close at 1731093205903Writing region close event to WAL at 1731093205903Closed at 1731093205903 2024-11-08T19:13:25,904 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 07363d44bad100b56e1b1aba358b0ed2, NAME => 'test,nnn,1731093205731.07363d44bad100b56e1b1aba358b0ed2.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f 2024-11-08T19:13:25,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741849_1025 (size=45) 2024-11-08T19:13:25,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741849_1025 (size=45) 2024-11-08T19:13:25,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741849_1025 (size=45) 2024-11-08T19:13:25,917 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,lll,1731093205731.6e461a86c03ca80ab184602fce3cc56c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:25,917 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 6e461a86c03ca80ab184602fce3cc56c, disabling compactions & flushes 2024-11-08T19:13:25,917 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,lll,1731093205731.6e461a86c03ca80ab184602fce3cc56c. 2024-11-08T19:13:25,917 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1731093205731.6e461a86c03ca80ab184602fce3cc56c. 2024-11-08T19:13:25,917 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1731093205731.6e461a86c03ca80ab184602fce3cc56c. after waiting 0 ms 2024-11-08T19:13:25,917 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1731093205731.6e461a86c03ca80ab184602fce3cc56c. 2024-11-08T19:13:25,917 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,lll,1731093205731.6e461a86c03ca80ab184602fce3cc56c. 2024-11-08T19:13:25,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741850_1026 (size=45) 2024-11-08T19:13:25,918 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 6e461a86c03ca80ab184602fce3cc56c: Waiting for close lock at 1731093205917Disabling compacts and flushes for region at 1731093205917Disabling writes for close at 1731093205917Writing region close event to WAL at 1731093205917Closed at 1731093205917 2024-11-08T19:13:25,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741850_1026 (size=45) 2024-11-08T19:13:25,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741850_1026 (size=45) 2024-11-08T19:13:25,918 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 0095b58ae56790b54bf50c65ac0754fa, NAME => 'test,ooo,1731093205731.0095b58ae56790b54bf50c65ac0754fa.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f 2024-11-08T19:13:25,919 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,mmm,1731093205731.245339f649708a4ea812249dd3f822c4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:25,919 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 245339f649708a4ea812249dd3f822c4, disabling compactions & flushes 2024-11-08T19:13:25,919 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,mmm,1731093205731.245339f649708a4ea812249dd3f822c4. 2024-11-08T19:13:25,919 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1731093205731.245339f649708a4ea812249dd3f822c4. 2024-11-08T19:13:25,919 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1731093205731.245339f649708a4ea812249dd3f822c4. after waiting 0 ms 2024-11-08T19:13:25,919 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1731093205731.245339f649708a4ea812249dd3f822c4. 2024-11-08T19:13:25,920 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,mmm,1731093205731.245339f649708a4ea812249dd3f822c4. 2024-11-08T19:13:25,920 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 245339f649708a4ea812249dd3f822c4: Waiting for close lock at 1731093205919Disabling compacts and flushes for region at 1731093205919Disabling writes for close at 1731093205919Writing region close event to WAL at 1731093205920 (+1 ms)Closed at 1731093205920 2024-11-08T19:13:25,920 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => e92192d7abb7a6c62f8815702d7dc38d, NAME => 'test,ppp,1731093205731.e92192d7abb7a6c62f8815702d7dc38d.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f 2024-11-08T19:13:25,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741851_1027 (size=45) 2024-11-08T19:13:25,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741851_1027 (size=45) 2024-11-08T19:13:25,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741851_1027 (size=45) 2024-11-08T19:13:25,924 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,nnn,1731093205731.07363d44bad100b56e1b1aba358b0ed2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:25,924 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 07363d44bad100b56e1b1aba358b0ed2, disabling compactions & flushes 2024-11-08T19:13:25,925 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,nnn,1731093205731.07363d44bad100b56e1b1aba358b0ed2. 2024-11-08T19:13:25,925 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1731093205731.07363d44bad100b56e1b1aba358b0ed2. 2024-11-08T19:13:25,925 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1731093205731.07363d44bad100b56e1b1aba358b0ed2. after waiting 0 ms 2024-11-08T19:13:25,925 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1731093205731.07363d44bad100b56e1b1aba358b0ed2. 2024-11-08T19:13:25,925 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,nnn,1731093205731.07363d44bad100b56e1b1aba358b0ed2. 2024-11-08T19:13:25,925 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 07363d44bad100b56e1b1aba358b0ed2: Waiting for close lock at 1731093205924Disabling compacts and flushes for region at 1731093205924Disabling writes for close at 1731093205925 (+1 ms)Writing region close event to WAL at 1731093205925Closed at 1731093205925 2024-11-08T19:13:25,926 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => caa4304cf54e018c1a1a2071373865b5, NAME => 'test,qqq,1731093205731.caa4304cf54e018c1a1a2071373865b5.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f 2024-11-08T19:13:25,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741852_1028 (size=45) 2024-11-08T19:13:25,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741852_1028 (size=45) 2024-11-08T19:13:25,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741852_1028 (size=45) 2024-11-08T19:13:25,935 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ooo,1731093205731.0095b58ae56790b54bf50c65ac0754fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:25,935 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 0095b58ae56790b54bf50c65ac0754fa, disabling compactions & flushes 2024-11-08T19:13:25,935 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ooo,1731093205731.0095b58ae56790b54bf50c65ac0754fa. 2024-11-08T19:13:25,935 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1731093205731.0095b58ae56790b54bf50c65ac0754fa. 2024-11-08T19:13:25,935 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1731093205731.0095b58ae56790b54bf50c65ac0754fa. after waiting 0 ms 2024-11-08T19:13:25,935 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1731093205731.0095b58ae56790b54bf50c65ac0754fa. 2024-11-08T19:13:25,935 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ooo,1731093205731.0095b58ae56790b54bf50c65ac0754fa. 2024-11-08T19:13:25,936 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 0095b58ae56790b54bf50c65ac0754fa: Waiting for close lock at 1731093205935Disabling compacts and flushes for region at 1731093205935Disabling writes for close at 1731093205935Writing region close event to WAL at 1731093205935Closed at 1731093205935 2024-11-08T19:13:25,936 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => f1496a2d5d6037cbe8d2f7a8f1407184, NAME => 'test,rrr,1731093205731.f1496a2d5d6037cbe8d2f7a8f1407184.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f 2024-11-08T19:13:25,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741853_1029 (size=45) 2024-11-08T19:13:25,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741853_1029 (size=45) 2024-11-08T19:13:25,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741853_1029 (size=45) 2024-11-08T19:13:25,948 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ppp,1731093205731.e92192d7abb7a6c62f8815702d7dc38d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:25,948 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing e92192d7abb7a6c62f8815702d7dc38d, disabling compactions & flushes 2024-11-08T19:13:25,948 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ppp,1731093205731.e92192d7abb7a6c62f8815702d7dc38d. 2024-11-08T19:13:25,948 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1731093205731.e92192d7abb7a6c62f8815702d7dc38d. 2024-11-08T19:13:25,948 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1731093205731.e92192d7abb7a6c62f8815702d7dc38d. after waiting 0 ms 2024-11-08T19:13:25,948 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1731093205731.e92192d7abb7a6c62f8815702d7dc38d. 2024-11-08T19:13:25,948 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ppp,1731093205731.e92192d7abb7a6c62f8815702d7dc38d. 2024-11-08T19:13:25,948 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for e92192d7abb7a6c62f8815702d7dc38d: Waiting for close lock at 1731093205948Disabling compacts and flushes for region at 1731093205948Disabling writes for close at 1731093205948Writing region close event to WAL at 1731093205948Closed at 1731093205948 2024-11-08T19:13:25,950 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 77e3781e625f0b456041758476a7d6b2, NAME => 'test,sss,1731093205731.77e3781e625f0b456041758476a7d6b2.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f 2024-11-08T19:13:25,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741854_1030 (size=45) 2024-11-08T19:13:25,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741854_1030 (size=45) 2024-11-08T19:13:25,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741854_1030 (size=45) 2024-11-08T19:13:25,958 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,qqq,1731093205731.caa4304cf54e018c1a1a2071373865b5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:25,958 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing caa4304cf54e018c1a1a2071373865b5, disabling compactions & flushes 2024-11-08T19:13:25,959 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,qqq,1731093205731.caa4304cf54e018c1a1a2071373865b5. 2024-11-08T19:13:25,959 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1731093205731.caa4304cf54e018c1a1a2071373865b5. 2024-11-08T19:13:25,959 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1731093205731.caa4304cf54e018c1a1a2071373865b5. after waiting 0 ms 2024-11-08T19:13:25,959 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1731093205731.caa4304cf54e018c1a1a2071373865b5. 2024-11-08T19:13:25,959 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,qqq,1731093205731.caa4304cf54e018c1a1a2071373865b5. 2024-11-08T19:13:25,959 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for caa4304cf54e018c1a1a2071373865b5: Waiting for close lock at 1731093205958Disabling compacts and flushes for region at 1731093205958Disabling writes for close at 1731093205959 (+1 ms)Writing region close event to WAL at 1731093205959Closed at 1731093205959 2024-11-08T19:13:25,960 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1918a8713e3d1c571d1188f35d75a101, NAME => 'test,ttt,1731093205731.1918a8713e3d1c571d1188f35d75a101.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f 2024-11-08T19:13:25,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741855_1031 (size=45) 2024-11-08T19:13:25,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741855_1031 (size=45) 2024-11-08T19:13:25,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741855_1031 (size=45) 2024-11-08T19:13:25,964 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,rrr,1731093205731.f1496a2d5d6037cbe8d2f7a8f1407184.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:25,965 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing f1496a2d5d6037cbe8d2f7a8f1407184, disabling compactions & flushes 2024-11-08T19:13:25,965 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,rrr,1731093205731.f1496a2d5d6037cbe8d2f7a8f1407184. 2024-11-08T19:13:25,965 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1731093205731.f1496a2d5d6037cbe8d2f7a8f1407184. 2024-11-08T19:13:25,965 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1731093205731.f1496a2d5d6037cbe8d2f7a8f1407184. after waiting 0 ms 2024-11-08T19:13:25,965 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1731093205731.f1496a2d5d6037cbe8d2f7a8f1407184. 2024-11-08T19:13:25,965 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,rrr,1731093205731.f1496a2d5d6037cbe8d2f7a8f1407184. 2024-11-08T19:13:25,965 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for f1496a2d5d6037cbe8d2f7a8f1407184: Waiting for close lock at 1731093205965Disabling compacts and flushes for region at 1731093205965Disabling writes for close at 1731093205965Writing region close event to WAL at 1731093205965Closed at 1731093205965 2024-11-08T19:13:25,966 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 234c6774a52fc6a092eb269a4a390dc7, NAME => 'test,uuu,1731093205731.234c6774a52fc6a092eb269a4a390dc7.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f 2024-11-08T19:13:25,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741856_1032 (size=45) 2024-11-08T19:13:25,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741856_1032 (size=45) 2024-11-08T19:13:25,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741856_1032 (size=45) 2024-11-08T19:13:25,977 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,sss,1731093205731.77e3781e625f0b456041758476a7d6b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:25,977 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 77e3781e625f0b456041758476a7d6b2, disabling compactions & flushes 2024-11-08T19:13:25,977 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,sss,1731093205731.77e3781e625f0b456041758476a7d6b2. 2024-11-08T19:13:25,977 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1731093205731.77e3781e625f0b456041758476a7d6b2. 2024-11-08T19:13:25,977 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1731093205731.77e3781e625f0b456041758476a7d6b2. after waiting 0 ms 2024-11-08T19:13:25,977 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1731093205731.77e3781e625f0b456041758476a7d6b2. 2024-11-08T19:13:25,977 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,sss,1731093205731.77e3781e625f0b456041758476a7d6b2. 2024-11-08T19:13:25,977 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 77e3781e625f0b456041758476a7d6b2: Waiting for close lock at 1731093205977Disabling compacts and flushes for region at 1731093205977Disabling writes for close at 1731093205977Writing region close event to WAL at 1731093205977Closed at 1731093205977 2024-11-08T19:13:25,978 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => f7c9c940725de9fafd49c71f7bebc530, NAME => 'test,vvv,1731093205731.f7c9c940725de9fafd49c71f7bebc530.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f 2024-11-08T19:13:25,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741857_1033 (size=45) 2024-11-08T19:13:25,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741857_1033 (size=45) 2024-11-08T19:13:25,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741857_1033 (size=45) 2024-11-08T19:13:25,984 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ttt,1731093205731.1918a8713e3d1c571d1188f35d75a101.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:25,984 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 1918a8713e3d1c571d1188f35d75a101, disabling compactions & flushes 2024-11-08T19:13:25,984 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ttt,1731093205731.1918a8713e3d1c571d1188f35d75a101. 2024-11-08T19:13:25,984 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1731093205731.1918a8713e3d1c571d1188f35d75a101. 2024-11-08T19:13:25,984 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1731093205731.1918a8713e3d1c571d1188f35d75a101. after waiting 0 ms 2024-11-08T19:13:25,984 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1731093205731.1918a8713e3d1c571d1188f35d75a101. 2024-11-08T19:13:25,984 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ttt,1731093205731.1918a8713e3d1c571d1188f35d75a101. 2024-11-08T19:13:25,984 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1918a8713e3d1c571d1188f35d75a101: Waiting for close lock at 1731093205984Disabling compacts and flushes for region at 1731093205984Disabling writes for close at 1731093205984Writing region close event to WAL at 1731093205984Closed at 1731093205984 2024-11-08T19:13:25,985 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9e50623b9f9d6b5fc998b958dd629458, NAME => 'test,www,1731093205731.9e50623b9f9d6b5fc998b958dd629458.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f 2024-11-08T19:13:25,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741858_1034 (size=45) 2024-11-08T19:13:25,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741858_1034 (size=45) 2024-11-08T19:13:25,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741858_1034 (size=45) 2024-11-08T19:13:25,993 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,uuu,1731093205731.234c6774a52fc6a092eb269a4a390dc7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:25,993 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 234c6774a52fc6a092eb269a4a390dc7, disabling compactions & flushes 2024-11-08T19:13:25,993 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,uuu,1731093205731.234c6774a52fc6a092eb269a4a390dc7. 2024-11-08T19:13:25,993 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1731093205731.234c6774a52fc6a092eb269a4a390dc7. 2024-11-08T19:13:25,993 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1731093205731.234c6774a52fc6a092eb269a4a390dc7. after waiting 0 ms 2024-11-08T19:13:25,993 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1731093205731.234c6774a52fc6a092eb269a4a390dc7. 2024-11-08T19:13:25,993 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,uuu,1731093205731.234c6774a52fc6a092eb269a4a390dc7. 2024-11-08T19:13:25,993 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 234c6774a52fc6a092eb269a4a390dc7: Waiting for close lock at 1731093205993Disabling compacts and flushes for region at 1731093205993Disabling writes for close at 1731093205993Writing region close event to WAL at 1731093205993Closed at 1731093205993 2024-11-08T19:13:25,994 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 48bb25f64ce0dca8172688cf9a77f57b, NAME => 'test,xxx,1731093205731.48bb25f64ce0dca8172688cf9a77f57b.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f 2024-11-08T19:13:25,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741859_1035 (size=45) 2024-11-08T19:13:25,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741859_1035 (size=45) 2024-11-08T19:13:25,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741859_1035 (size=45) 2024-11-08T19:13:25,996 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,vvv,1731093205731.f7c9c940725de9fafd49c71f7bebc530.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:25,996 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing f7c9c940725de9fafd49c71f7bebc530, disabling compactions & flushes 2024-11-08T19:13:25,996 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,vvv,1731093205731.f7c9c940725de9fafd49c71f7bebc530. 2024-11-08T19:13:25,996 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1731093205731.f7c9c940725de9fafd49c71f7bebc530. 2024-11-08T19:13:25,996 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1731093205731.f7c9c940725de9fafd49c71f7bebc530. after waiting 0 ms 2024-11-08T19:13:25,996 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1731093205731.f7c9c940725de9fafd49c71f7bebc530. 2024-11-08T19:13:25,996 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,vvv,1731093205731.f7c9c940725de9fafd49c71f7bebc530. 2024-11-08T19:13:25,996 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for f7c9c940725de9fafd49c71f7bebc530: Waiting for close lock at 1731093205996Disabling compacts and flushes for region at 1731093205996Disabling writes for close at 1731093205996Writing region close event to WAL at 1731093205996Closed at 1731093205996 2024-11-08T19:13:25,997 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 22cccf3cc4c2d3343ceae07ff16f899f, NAME => 'test,yyy,1731093205731.22cccf3cc4c2d3343ceae07ff16f899f.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f 2024-11-08T19:13:25,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741860_1036 (size=45) 2024-11-08T19:13:25,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741860_1036 (size=45) 2024-11-08T19:13:25,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741860_1036 (size=45) 2024-11-08T19:13:26,000 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,www,1731093205731.9e50623b9f9d6b5fc998b958dd629458.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:26,000 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 9e50623b9f9d6b5fc998b958dd629458, disabling compactions & flushes 2024-11-08T19:13:26,000 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,www,1731093205731.9e50623b9f9d6b5fc998b958dd629458. 2024-11-08T19:13:26,000 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1731093205731.9e50623b9f9d6b5fc998b958dd629458. 2024-11-08T19:13:26,000 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1731093205731.9e50623b9f9d6b5fc998b958dd629458. after waiting 0 ms 2024-11-08T19:13:26,000 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1731093205731.9e50623b9f9d6b5fc998b958dd629458. 2024-11-08T19:13:26,000 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,www,1731093205731.9e50623b9f9d6b5fc998b958dd629458. 2024-11-08T19:13:26,000 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9e50623b9f9d6b5fc998b958dd629458: Waiting for close lock at 1731093206000Disabling compacts and flushes for region at 1731093206000Disabling writes for close at 1731093206000Writing region close event to WAL at 1731093206000Closed at 1731093206000 2024-11-08T19:13:26,001 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2afe1e3e862a406c7fcb359dc1b60d57, NAME => 'test,zzz,1731093205731.2afe1e3e862a406c7fcb359dc1b60d57.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f 2024-11-08T19:13:26,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741861_1037 (size=45) 2024-11-08T19:13:26,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741862_1038 (size=45) 2024-11-08T19:13:26,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741861_1037 (size=45) 2024-11-08T19:13:26,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741861_1037 (size=45) 2024-11-08T19:13:26,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741862_1038 (size=45) 2024-11-08T19:13:26,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741862_1038 (size=45) 2024-11-08T19:13:26,010 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,yyy,1731093205731.22cccf3cc4c2d3343ceae07ff16f899f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:26,010 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 22cccf3cc4c2d3343ceae07ff16f899f, disabling compactions & flushes 2024-11-08T19:13:26,010 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,yyy,1731093205731.22cccf3cc4c2d3343ceae07ff16f899f. 2024-11-08T19:13:26,010 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1731093205731.22cccf3cc4c2d3343ceae07ff16f899f. 2024-11-08T19:13:26,010 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1731093205731.22cccf3cc4c2d3343ceae07ff16f899f. after waiting 0 ms 2024-11-08T19:13:26,010 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1731093205731.22cccf3cc4c2d3343ceae07ff16f899f. 2024-11-08T19:13:26,010 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,yyy,1731093205731.22cccf3cc4c2d3343ceae07ff16f899f. 2024-11-08T19:13:26,010 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 22cccf3cc4c2d3343ceae07ff16f899f: Waiting for close lock at 1731093206010Disabling compacts and flushes for region at 1731093206010Disabling writes for close at 1731093206010Writing region close event to WAL at 1731093206010Closed at 1731093206010 2024-11-08T19:13:26,011 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,xxx,1731093205731.48bb25f64ce0dca8172688cf9a77f57b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:26,011 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 48bb25f64ce0dca8172688cf9a77f57b, disabling compactions & flushes 2024-11-08T19:13:26,012 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,xxx,1731093205731.48bb25f64ce0dca8172688cf9a77f57b. 2024-11-08T19:13:26,012 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1731093205731.48bb25f64ce0dca8172688cf9a77f57b. 2024-11-08T19:13:26,012 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1731093205731.48bb25f64ce0dca8172688cf9a77f57b. after waiting 0 ms 2024-11-08T19:13:26,012 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1731093205731.48bb25f64ce0dca8172688cf9a77f57b. 2024-11-08T19:13:26,012 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,xxx,1731093205731.48bb25f64ce0dca8172688cf9a77f57b. 2024-11-08T19:13:26,012 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 48bb25f64ce0dca8172688cf9a77f57b: Waiting for close lock at 1731093206011Disabling compacts and flushes for region at 1731093206011Disabling writes for close at 1731093206012 (+1 ms)Writing region close event to WAL at 1731093206012Closed at 1731093206012 2024-11-08T19:13:26,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741863_1039 (size=42) 2024-11-08T19:13:26,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741863_1039 (size=42) 2024-11-08T19:13:26,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741863_1039 (size=42) 2024-11-08T19:13:26,016 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,zzz,1731093205731.2afe1e3e862a406c7fcb359dc1b60d57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:26,016 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 2afe1e3e862a406c7fcb359dc1b60d57, disabling compactions & flushes 2024-11-08T19:13:26,016 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,zzz,1731093205731.2afe1e3e862a406c7fcb359dc1b60d57. 2024-11-08T19:13:26,016 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1731093205731.2afe1e3e862a406c7fcb359dc1b60d57. 2024-11-08T19:13:26,016 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1731093205731.2afe1e3e862a406c7fcb359dc1b60d57. after waiting 0 ms 2024-11-08T19:13:26,016 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1731093205731.2afe1e3e862a406c7fcb359dc1b60d57. 2024-11-08T19:13:26,016 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,zzz,1731093205731.2afe1e3e862a406c7fcb359dc1b60d57. 2024-11-08T19:13:26,016 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2afe1e3e862a406c7fcb359dc1b60d57: Waiting for close lock at 1731093206016Disabling compacts and flushes for region at 1731093206016Disabling writes for close at 1731093206016Writing region close event to WAL at 1731093206016Closed at 1731093206016 2024-11-08T19:13:26,019 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T19:13:26,026 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1731093205731.7b4a23d5efe8e832c0d5cb16df3b1369.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731093206019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731093206019"}]},"ts":"1731093206019"} 2024-11-08T19:13:26,026 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1731093205731.d2ea8472f395c484d4998844c4dcd304.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731093206019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731093206019"}]},"ts":"1731093206019"} 2024-11-08T19:13:26,027 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1731093205731.8cb2ac5239e07e076f513604c29c2ca8.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731093206019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731093206019"}]},"ts":"1731093206019"} 2024-11-08T19:13:26,027 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1731093205731.a8e891c0073a08913834761f455a77e8.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731093206019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731093206019"}]},"ts":"1731093206019"} 2024-11-08T19:13:26,027 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1731093205731.fdc84e1c2d9bdd5f39d647a57ff91a05.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731093206019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731093206019"}]},"ts":"1731093206019"} 2024-11-08T19:13:26,027 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1731093205731.53b7061bfce2939f050c57221eed7f2a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731093206019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731093206019"}]},"ts":"1731093206019"} 2024-11-08T19:13:26,028 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1731093205731.ab92690881683c86166f7f23976b5188.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731093206019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731093206019"}]},"ts":"1731093206019"} 2024-11-08T19:13:26,028 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1731093205731.7d1d9fe849fd45b830c330834b58d508.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731093206019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731093206019"}]},"ts":"1731093206019"} 2024-11-08T19:13:26,028 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1731093205731.2548b6c044cd2c9ceb7dec340e2cde6c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731093206019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731093206019"}]},"ts":"1731093206019"} 2024-11-08T19:13:26,028 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1731093205731.f7a8c4f9e9448f30c950ddcf996ae56f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731093206019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731093206019"}]},"ts":"1731093206019"} 2024-11-08T19:13:26,029 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1731093205731.3d0557ff953666f2f97a7be0e2d0bc87.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731093206019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731093206019"}]},"ts":"1731093206019"} 2024-11-08T19:13:26,029 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1731093205731.6e461a86c03ca80ab184602fce3cc56c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731093206019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731093206019"}]},"ts":"1731093206019"} 2024-11-08T19:13:26,029 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1731093205731.245339f649708a4ea812249dd3f822c4.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731093206019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731093206019"}]},"ts":"1731093206019"} 2024-11-08T19:13:26,029 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1731093205731.07363d44bad100b56e1b1aba358b0ed2.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731093206019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731093206019"}]},"ts":"1731093206019"} 2024-11-08T19:13:26,029 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1731093205731.0095b58ae56790b54bf50c65ac0754fa.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731093206019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731093206019"}]},"ts":"1731093206019"} 2024-11-08T19:13:26,029 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1731093205731.e92192d7abb7a6c62f8815702d7dc38d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731093206019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731093206019"}]},"ts":"1731093206019"} 2024-11-08T19:13:26,030 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1731093205731.caa4304cf54e018c1a1a2071373865b5.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731093206019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731093206019"}]},"ts":"1731093206019"} 2024-11-08T19:13:26,030 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1731093205731.f1496a2d5d6037cbe8d2f7a8f1407184.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731093206019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731093206019"}]},"ts":"1731093206019"} 2024-11-08T19:13:26,030 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1731093205731.77e3781e625f0b456041758476a7d6b2.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731093206019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731093206019"}]},"ts":"1731093206019"} 2024-11-08T19:13:26,030 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1731093205731.1918a8713e3d1c571d1188f35d75a101.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731093206019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731093206019"}]},"ts":"1731093206019"} 2024-11-08T19:13:26,031 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1731093205731.234c6774a52fc6a092eb269a4a390dc7.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731093206019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731093206019"}]},"ts":"1731093206019"} 2024-11-08T19:13:26,031 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1731093205731.f7c9c940725de9fafd49c71f7bebc530.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731093206019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731093206019"}]},"ts":"1731093206019"} 2024-11-08T19:13:26,031 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1731093205731.9e50623b9f9d6b5fc998b958dd629458.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731093206019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731093206019"}]},"ts":"1731093206019"} 2024-11-08T19:13:26,031 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1731093205731.22cccf3cc4c2d3343ceae07ff16f899f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731093206019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731093206019"}]},"ts":"1731093206019"} 2024-11-08T19:13:26,031 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1731093205731.48bb25f64ce0dca8172688cf9a77f57b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731093206019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731093206019"}]},"ts":"1731093206019"} 2024-11-08T19:13:26,032 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1731093205731.2afe1e3e862a406c7fcb359dc1b60d57.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731093206019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731093206019"}]},"ts":"1731093206019"} 2024-11-08T19:13:26,078 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-08T19:13:26,081 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T19:13:26,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T19:13:26,083 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731093206081"}]},"ts":"1731093206081"} 2024-11-08T19:13:26,088 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-11-08T19:13:26,089 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {692b295ad45c=0} racks are {/default-rack=0} 2024-11-08T19:13:26,105 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T19:13:26,105 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T19:13:26,105 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-08T19:13:26,105 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T19:13:26,105 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T19:13:26,105 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-08T19:13:26,105 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T19:13:26,105 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T19:13:26,105 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-08T19:13:26,105 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T19:13:26,107 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d2ea8472f395c484d4998844c4dcd304, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7b4a23d5efe8e832c0d5cb16df3b1369, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a8e891c0073a08913834761f455a77e8, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fdc84e1c2d9bdd5f39d647a57ff91a05, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ab92690881683c86166f7f23976b5188, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7d1d9fe849fd45b830c330834b58d508, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f7a8c4f9e9448f30c950ddcf996ae56f, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3d0557ff953666f2f97a7be0e2d0bc87, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=245339f649708a4ea812249dd3f822c4, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=07363d44bad100b56e1b1aba358b0ed2, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e92192d7abb7a6c62f8815702d7dc38d, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=caa4304cf54e018c1a1a2071373865b5, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=77e3781e625f0b456041758476a7d6b2, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1918a8713e3d1c571d1188f35d75a101, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f7c9c940725de9fafd49c71f7bebc530, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9e50623b9f9d6b5fc998b958dd629458, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=48bb25f64ce0dca8172688cf9a77f57b, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2afe1e3e862a406c7fcb359dc1b60d57, ASSIGN}] 2024-11-08T19:13:26,116 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2afe1e3e862a406c7fcb359dc1b60d57, ASSIGN 2024-11-08T19:13:26,118 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2afe1e3e862a406c7fcb359dc1b60d57, ASSIGN; state=OFFLINE, location=692b295ad45c,36007,1731093202744; forceNewPlan=false, retain=false 2024-11-08T19:13:26,119 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN 2024-11-08T19:13:26,121 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN; state=OFFLINE, location=692b295ad45c,39647,1731093203061; forceNewPlan=false, retain=false 2024-11-08T19:13:26,123 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=48bb25f64ce0dca8172688cf9a77f57b, ASSIGN 2024-11-08T19:13:26,124 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=48bb25f64ce0dca8172688cf9a77f57b, ASSIGN; state=OFFLINE, location=692b295ad45c,42693,1731093202977; forceNewPlan=false, retain=false 2024-11-08T19:13:26,126 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9e50623b9f9d6b5fc998b958dd629458, ASSIGN 2024-11-08T19:13:26,127 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9e50623b9f9d6b5fc998b958dd629458, ASSIGN; state=OFFLINE, location=692b295ad45c,36007,1731093202744; forceNewPlan=false, retain=false 2024-11-08T19:13:26,128 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f7c9c940725de9fafd49c71f7bebc530, ASSIGN 2024-11-08T19:13:26,130 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f7c9c940725de9fafd49c71f7bebc530, ASSIGN; state=OFFLINE, location=692b295ad45c,42693,1731093202977; forceNewPlan=false, retain=false 2024-11-08T19:13:26,132 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN 2024-11-08T19:13:26,133 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN; state=OFFLINE, location=692b295ad45c,39647,1731093203061; forceNewPlan=false, retain=false 2024-11-08T19:13:26,135 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1918a8713e3d1c571d1188f35d75a101, ASSIGN 2024-11-08T19:13:26,137 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1918a8713e3d1c571d1188f35d75a101, ASSIGN; state=OFFLINE, location=692b295ad45c,36007,1731093202744; forceNewPlan=false, retain=false 2024-11-08T19:13:26,138 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=77e3781e625f0b456041758476a7d6b2, ASSIGN 2024-11-08T19:13:26,140 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=77e3781e625f0b456041758476a7d6b2, ASSIGN; state=OFFLINE, location=692b295ad45c,42693,1731093202977; forceNewPlan=false, retain=false 2024-11-08T19:13:26,142 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN 2024-11-08T19:13:26,143 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN; state=OFFLINE, location=692b295ad45c,39647,1731093203061; forceNewPlan=false, retain=false 2024-11-08T19:13:26,144 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=caa4304cf54e018c1a1a2071373865b5, ASSIGN 2024-11-08T19:13:26,146 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=caa4304cf54e018c1a1a2071373865b5, ASSIGN; state=OFFLINE, location=692b295ad45c,36007,1731093202744; forceNewPlan=false, retain=false 2024-11-08T19:13:26,147 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e92192d7abb7a6c62f8815702d7dc38d, ASSIGN 2024-11-08T19:13:26,148 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e92192d7abb7a6c62f8815702d7dc38d, ASSIGN; state=OFFLINE, location=692b295ad45c,42693,1731093202977; forceNewPlan=false, retain=false 2024-11-08T19:13:26,150 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN 2024-11-08T19:13:26,151 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN; state=OFFLINE, location=692b295ad45c,39647,1731093203061; forceNewPlan=false, retain=false 2024-11-08T19:13:26,153 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=07363d44bad100b56e1b1aba358b0ed2, ASSIGN 2024-11-08T19:13:26,154 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=07363d44bad100b56e1b1aba358b0ed2, ASSIGN; state=OFFLINE, location=692b295ad45c,36007,1731093202744; forceNewPlan=false, retain=false 2024-11-08T19:13:26,156 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=245339f649708a4ea812249dd3f822c4, ASSIGN 2024-11-08T19:13:26,157 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=245339f649708a4ea812249dd3f822c4, ASSIGN; state=OFFLINE, location=692b295ad45c,42693,1731093202977; forceNewPlan=false, retain=false 2024-11-08T19:13:26,159 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN 2024-11-08T19:13:26,160 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN; state=OFFLINE, location=692b295ad45c,39647,1731093203061; forceNewPlan=false, retain=false 2024-11-08T19:13:26,161 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3d0557ff953666f2f97a7be0e2d0bc87, ASSIGN 2024-11-08T19:13:26,163 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=3d0557ff953666f2f97a7be0e2d0bc87, ASSIGN; state=OFFLINE, location=692b295ad45c,36007,1731093202744; forceNewPlan=false, retain=false 2024-11-08T19:13:26,164 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f7a8c4f9e9448f30c950ddcf996ae56f, ASSIGN 2024-11-08T19:13:26,165 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f7a8c4f9e9448f30c950ddcf996ae56f, ASSIGN; state=OFFLINE, location=692b295ad45c,42693,1731093202977; forceNewPlan=false, retain=false 2024-11-08T19:13:26,166 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN 2024-11-08T19:13:26,168 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN; state=OFFLINE, location=692b295ad45c,39647,1731093203061; forceNewPlan=false, retain=false 2024-11-08T19:13:26,169 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7d1d9fe849fd45b830c330834b58d508, ASSIGN 2024-11-08T19:13:26,170 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7d1d9fe849fd45b830c330834b58d508, ASSIGN; state=OFFLINE, location=692b295ad45c,36007,1731093202744; forceNewPlan=false, retain=false 2024-11-08T19:13:26,172 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ab92690881683c86166f7f23976b5188, ASSIGN 2024-11-08T19:13:26,173 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ab92690881683c86166f7f23976b5188, ASSIGN; state=OFFLINE, location=692b295ad45c,42693,1731093202977; forceNewPlan=false, retain=false 2024-11-08T19:13:26,174 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN 2024-11-08T19:13:26,175 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN; state=OFFLINE, location=692b295ad45c,39647,1731093203061; forceNewPlan=false, retain=false 2024-11-08T19:13:26,177 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fdc84e1c2d9bdd5f39d647a57ff91a05, ASSIGN 2024-11-08T19:13:26,178 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=fdc84e1c2d9bdd5f39d647a57ff91a05, ASSIGN; state=OFFLINE, location=692b295ad45c,36007,1731093202744; forceNewPlan=false, retain=false 2024-11-08T19:13:26,179 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a8e891c0073a08913834761f455a77e8, ASSIGN 2024-11-08T19:13:26,182 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a8e891c0073a08913834761f455a77e8, ASSIGN; state=OFFLINE, location=692b295ad45c,42693,1731093202977; forceNewPlan=false, retain=false 2024-11-08T19:13:26,184 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7b4a23d5efe8e832c0d5cb16df3b1369, ASSIGN 2024-11-08T19:13:26,186 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7b4a23d5efe8e832c0d5cb16df3b1369, ASSIGN; state=OFFLINE, location=692b295ad45c,42693,1731093202977; forceNewPlan=false, retain=false 2024-11-08T19:13:26,188 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN 2024-11-08T19:13:26,190 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN; state=OFFLINE, location=692b295ad45c,39647,1731093203061; forceNewPlan=false, retain=false 2024-11-08T19:13:26,192 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d2ea8472f395c484d4998844c4dcd304, ASSIGN 2024-11-08T19:13:26,193 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d2ea8472f395c484d4998844c4dcd304, ASSIGN; state=OFFLINE, location=692b295ad45c,36007,1731093202744; forceNewPlan=false, retain=false 2024-11-08T19:13:26,271 INFO [692b295ad45c:39509 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-08T19:13:26,273 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=48bb25f64ce0dca8172688cf9a77f57b, regionState=OPENING, regionLocation=692b295ad45c,42693,1731093202977 2024-11-08T19:13:26,274 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=f7c9c940725de9fafd49c71f7bebc530, regionState=OPENING, regionLocation=692b295ad45c,42693,1731093202977 2024-11-08T19:13:26,275 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=77e3781e625f0b456041758476a7d6b2, regionState=OPENING, regionLocation=692b295ad45c,42693,1731093202977 2024-11-08T19:13:26,276 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=e92192d7abb7a6c62f8815702d7dc38d, regionState=OPENING, regionLocation=692b295ad45c,42693,1731093202977 2024-11-08T19:13:26,277 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=245339f649708a4ea812249dd3f822c4, regionState=OPENING, regionLocation=692b295ad45c,42693,1731093202977 2024-11-08T19:13:26,278 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=f7a8c4f9e9448f30c950ddcf996ae56f, regionState=OPENING, regionLocation=692b295ad45c,42693,1731093202977 2024-11-08T19:13:26,279 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=ab92690881683c86166f7f23976b5188, regionState=OPENING, regionLocation=692b295ad45c,42693,1731093202977 2024-11-08T19:13:26,280 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=a8e891c0073a08913834761f455a77e8, regionState=OPENING, regionLocation=692b295ad45c,42693,1731093202977 2024-11-08T19:13:26,281 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=48bb25f64ce0dca8172688cf9a77f57b, ASSIGN because future has completed 2024-11-08T19:13:26,281 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=7b4a23d5efe8e832c0d5cb16df3b1369, regionState=OPENING, regionLocation=692b295ad45c,42693,1731093202977 2024-11-08T19:13:26,282 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 48bb25f64ce0dca8172688cf9a77f57b, server=692b295ad45c,42693,1731093202977}] 2024-11-08T19:13:26,282 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f7c9c940725de9fafd49c71f7bebc530, ASSIGN because future has completed 2024-11-08T19:13:26,288 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=77e3781e625f0b456041758476a7d6b2, ASSIGN because future has completed 2024-11-08T19:13:26,290 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e92192d7abb7a6c62f8815702d7dc38d, ASSIGN because future has completed 2024-11-08T19:13:26,292 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=245339f649708a4ea812249dd3f822c4, ASSIGN because future has completed 2024-11-08T19:13:26,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 245339f649708a4ea812249dd3f822c4, server=692b295ad45c,42693,1731093202977}] 2024-11-08T19:13:26,294 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f7a8c4f9e9448f30c950ddcf996ae56f, ASSIGN because future has completed 2024-11-08T19:13:26,296 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ab92690881683c86166f7f23976b5188, ASSIGN because future has completed 2024-11-08T19:13:26,298 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a8e891c0073a08913834761f455a77e8, ASSIGN because future has completed 2024-11-08T19:13:26,301 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7b4a23d5efe8e832c0d5cb16df3b1369, ASSIGN because future has completed 2024-11-08T19:13:26,302 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7b4a23d5efe8e832c0d5cb16df3b1369, server=692b295ad45c,42693,1731093202977}] 2024-11-08T19:13:26,309 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure a8e891c0073a08913834761f455a77e8, server=692b295ad45c,42693,1731093202977}] 2024-11-08T19:13:26,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab92690881683c86166f7f23976b5188, server=692b295ad45c,42693,1731093202977}] 2024-11-08T19:13:26,321 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure f7a8c4f9e9448f30c950ddcf996ae56f, server=692b295ad45c,42693,1731093202977}] 2024-11-08T19:13:26,326 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure e92192d7abb7a6c62f8815702d7dc38d, server=692b295ad45c,42693,1731093202977}] 2024-11-08T19:13:26,332 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 77e3781e625f0b456041758476a7d6b2, server=692b295ad45c,42693,1731093202977}] 2024-11-08T19:13:26,337 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure f7c9c940725de9fafd49c71f7bebc530, server=692b295ad45c,42693,1731093202977}] 2024-11-08T19:13:26,342 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=22cccf3cc4c2d3343ceae07ff16f899f, regionState=OPENING, regionLocation=692b295ad45c,39647,1731093203061 2024-11-08T19:13:26,343 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=234c6774a52fc6a092eb269a4a390dc7, regionState=OPENING, regionLocation=692b295ad45c,39647,1731093203061 2024-11-08T19:13:26,343 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=f1496a2d5d6037cbe8d2f7a8f1407184, regionState=OPENING, regionLocation=692b295ad45c,39647,1731093203061 2024-11-08T19:13:26,344 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=0095b58ae56790b54bf50c65ac0754fa, regionState=OPENING, regionLocation=692b295ad45c,39647,1731093203061 2024-11-08T19:13:26,345 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=6e461a86c03ca80ab184602fce3cc56c, regionState=OPENING, regionLocation=692b295ad45c,39647,1731093203061 2024-11-08T19:13:26,345 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=2548b6c044cd2c9ceb7dec340e2cde6c, regionState=OPENING, regionLocation=692b295ad45c,39647,1731093203061 2024-11-08T19:13:26,346 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=53b7061bfce2939f050c57221eed7f2a, regionState=OPENING, regionLocation=692b295ad45c,39647,1731093203061 2024-11-08T19:13:26,347 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN because future has completed 2024-11-08T19:13:26,347 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=8cb2ac5239e07e076f513604c29c2ca8, regionState=OPENING, regionLocation=692b295ad45c,39647,1731093203061 2024-11-08T19:13:26,347 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22cccf3cc4c2d3343ceae07ff16f899f, server=692b295ad45c,39647,1731093203061}] 2024-11-08T19:13:26,348 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN because future has completed 2024-11-08T19:13:26,350 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN because future has completed 2024-11-08T19:13:26,352 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN because future has completed 2024-11-08T19:13:26,355 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN because future has completed 2024-11-08T19:13:26,356 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6e461a86c03ca80ab184602fce3cc56c, server=692b295ad45c,39647,1731093203061}] 2024-11-08T19:13:26,357 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN because future has completed 2024-11-08T19:13:26,358 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN because future has completed 2024-11-08T19:13:26,360 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN because future has completed 2024-11-08T19:13:26,362 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,39647,1731093203061}] 2024-11-08T19:13:26,365 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 53b7061bfce2939f050c57221eed7f2a, server=692b295ad45c,39647,1731093203061}] 2024-11-08T19:13:26,369 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2548b6c044cd2c9ceb7dec340e2cde6c, server=692b295ad45c,39647,1731093203061}] 2024-11-08T19:13:26,374 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0095b58ae56790b54bf50c65ac0754fa, server=692b295ad45c,39647,1731093203061}] 2024-11-08T19:13:26,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1496a2d5d6037cbe8d2f7a8f1407184, server=692b295ad45c,39647,1731093203061}] 2024-11-08T19:13:26,384 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 234c6774a52fc6a092eb269a4a390dc7, server=692b295ad45c,39647,1731093203061}] 2024-11-08T19:13:26,388 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=2afe1e3e862a406c7fcb359dc1b60d57, regionState=OPENING, regionLocation=692b295ad45c,36007,1731093202744 2024-11-08T19:13:26,389 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=9e50623b9f9d6b5fc998b958dd629458, regionState=OPENING, regionLocation=692b295ad45c,36007,1731093202744 2024-11-08T19:13:26,390 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=1918a8713e3d1c571d1188f35d75a101, regionState=OPENING, regionLocation=692b295ad45c,36007,1731093202744 2024-11-08T19:13:26,390 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=caa4304cf54e018c1a1a2071373865b5, regionState=OPENING, regionLocation=692b295ad45c,36007,1731093202744 2024-11-08T19:13:26,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T19:13:26,391 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=07363d44bad100b56e1b1aba358b0ed2, regionState=OPENING, regionLocation=692b295ad45c,36007,1731093202744 2024-11-08T19:13:26,392 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=3d0557ff953666f2f97a7be0e2d0bc87, regionState=OPENING, regionLocation=692b295ad45c,36007,1731093202744 2024-11-08T19:13:26,392 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2afe1e3e862a406c7fcb359dc1b60d57, ASSIGN because future has completed 2024-11-08T19:13:26,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2afe1e3e862a406c7fcb359dc1b60d57, server=692b295ad45c,36007,1731093202744}] 2024-11-08T19:13:26,394 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9e50623b9f9d6b5fc998b958dd629458, ASSIGN because future has completed 2024-11-08T19:13:26,396 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1918a8713e3d1c571d1188f35d75a101, ASSIGN because future has completed 2024-11-08T19:13:26,398 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=caa4304cf54e018c1a1a2071373865b5, ASSIGN because future has completed 2024-11-08T19:13:26,399 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure caa4304cf54e018c1a1a2071373865b5, server=692b295ad45c,36007,1731093202744}] 2024-11-08T19:13:26,399 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=07363d44bad100b56e1b1aba358b0ed2, ASSIGN because future has completed 2024-11-08T19:13:26,401 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=3d0557ff953666f2f97a7be0e2d0bc87, ASSIGN because future has completed 2024-11-08T19:13:26,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3d0557ff953666f2f97a7be0e2d0bc87, server=692b295ad45c,36007,1731093202744}] 2024-11-08T19:13:26,408 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07363d44bad100b56e1b1aba358b0ed2, server=692b295ad45c,36007,1731093202744}] 2024-11-08T19:13:26,412 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1918a8713e3d1c571d1188f35d75a101, server=692b295ad45c,36007,1731093202744}] 2024-11-08T19:13:26,416 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9e50623b9f9d6b5fc998b958dd629458, server=692b295ad45c,36007,1731093202744}] 2024-11-08T19:13:26,420 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=7d1d9fe849fd45b830c330834b58d508, regionState=OPENING, regionLocation=692b295ad45c,36007,1731093202744 2024-11-08T19:13:26,421 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=fdc84e1c2d9bdd5f39d647a57ff91a05, regionState=OPENING, regionLocation=692b295ad45c,36007,1731093202744 2024-11-08T19:13:26,421 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d2ea8472f395c484d4998844c4dcd304, regionState=OPENING, regionLocation=692b295ad45c,36007,1731093202744 2024-11-08T19:13:26,423 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7d1d9fe849fd45b830c330834b58d508, ASSIGN because future has completed 2024-11-08T19:13:26,423 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7d1d9fe849fd45b830c330834b58d508, server=692b295ad45c,36007,1731093202744}] 2024-11-08T19:13:26,425 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=fdc84e1c2d9bdd5f39d647a57ff91a05, ASSIGN because future has completed 2024-11-08T19:13:26,427 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d2ea8472f395c484d4998844c4dcd304, ASSIGN because future has completed 2024-11-08T19:13:26,428 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2ea8472f395c484d4998844c4dcd304, server=692b295ad45c,36007,1731093202744}] 2024-11-08T19:13:26,431 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure fdc84e1c2d9bdd5f39d647a57ff91a05, server=692b295ad45c,36007,1731093202744}] 2024-11-08T19:13:26,440 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T19:13:26,442 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48605, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T19:13:26,447 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,mmm,1731093205731.245339f649708a4ea812249dd3f822c4. 2024-11-08T19:13:26,448 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 245339f649708a4ea812249dd3f822c4, NAME => 'test,mmm,1731093205731.245339f649708a4ea812249dd3f822c4.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-08T19:13:26,448 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 245339f649708a4ea812249dd3f822c4 2024-11-08T19:13:26,448 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,mmm,1731093205731.245339f649708a4ea812249dd3f822c4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:26,449 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 245339f649708a4ea812249dd3f822c4 2024-11-08T19:13:26,449 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 245339f649708a4ea812249dd3f822c4 2024-11-08T19:13:26,451 INFO [StoreOpener-245339f649708a4ea812249dd3f822c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 245339f649708a4ea812249dd3f822c4 2024-11-08T19:13:26,452 INFO [StoreOpener-245339f649708a4ea812249dd3f822c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 245339f649708a4ea812249dd3f822c4 columnFamilyName family 2024-11-08T19:13:26,452 DEBUG [StoreOpener-245339f649708a4ea812249dd3f822c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:26,453 INFO [StoreOpener-245339f649708a4ea812249dd3f822c4-1 {}] regionserver.HStore(327): Store=245339f649708a4ea812249dd3f822c4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T19:13:26,453 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 245339f649708a4ea812249dd3f822c4 2024-11-08T19:13:26,454 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/245339f649708a4ea812249dd3f822c4 2024-11-08T19:13:26,455 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/245339f649708a4ea812249dd3f822c4 2024-11-08T19:13:26,455 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 245339f649708a4ea812249dd3f822c4 2024-11-08T19:13:26,455 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 245339f649708a4ea812249dd3f822c4 2024-11-08T19:13:26,458 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 245339f649708a4ea812249dd3f822c4 2024-11-08T19:13:26,461 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/245339f649708a4ea812249dd3f822c4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T19:13:26,462 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 245339f649708a4ea812249dd3f822c4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75011774, jitterRate=0.11776253581047058}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T19:13:26,462 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 245339f649708a4ea812249dd3f822c4 2024-11-08T19:13:26,464 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 245339f649708a4ea812249dd3f822c4: Running coprocessor pre-open hook at 1731093206449Writing region info on filesystem at 1731093206449Initializing all the Stores at 1731093206450 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731093206450Cleaning up temporary data from old regions at 1731093206455 (+5 ms)Running coprocessor post-open hooks at 1731093206462 (+7 ms)Region opened successfully at 1731093206463 (+1 ms) 2024-11-08T19:13:26,465 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1731093205731.245339f649708a4ea812249dd3f822c4., pid=32, masterSystemTime=1731093206439 2024-11-08T19:13:26,468 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1731093205731.245339f649708a4ea812249dd3f822c4. 2024-11-08T19:13:26,468 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,mmm,1731093205731.245339f649708a4ea812249dd3f822c4. 2024-11-08T19:13:26,469 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,ddd,1731093205731.a8e891c0073a08913834761f455a77e8. 2024-11-08T19:13:26,469 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => a8e891c0073a08913834761f455a77e8, NAME => 'test,ddd,1731093205731.a8e891c0073a08913834761f455a77e8.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-08T19:13:26,469 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=245339f649708a4ea812249dd3f822c4, regionState=OPEN, openSeqNum=2, regionLocation=692b295ad45c,42693,1731093202977 2024-11-08T19:13:26,469 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a8e891c0073a08913834761f455a77e8 2024-11-08T19:13:26,470 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,ddd,1731093205731.a8e891c0073a08913834761f455a77e8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:26,470 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for a8e891c0073a08913834761f455a77e8 2024-11-08T19:13:26,470 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for a8e891c0073a08913834761f455a77e8 2024-11-08T19:13:26,473 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 245339f649708a4ea812249dd3f822c4, server=692b295ad45c,42693,1731093202977 because future has completed 2024-11-08T19:13:26,474 INFO [StoreOpener-a8e891c0073a08913834761f455a77e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a8e891c0073a08913834761f455a77e8 2024-11-08T19:13:26,476 INFO [StoreOpener-a8e891c0073a08913834761f455a77e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a8e891c0073a08913834761f455a77e8 columnFamilyName family 2024-11-08T19:13:26,476 DEBUG [StoreOpener-a8e891c0073a08913834761f455a77e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:26,477 INFO [StoreOpener-a8e891c0073a08913834761f455a77e8-1 {}] regionserver.HStore(327): Store=a8e891c0073a08913834761f455a77e8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T19:13:26,478 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for a8e891c0073a08913834761f455a77e8 2024-11-08T19:13:26,479 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/a8e891c0073a08913834761f455a77e8 2024-11-08T19:13:26,479 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=17 2024-11-08T19:13:26,479 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 245339f649708a4ea812249dd3f822c4, server=692b295ad45c,42693,1731093202977 in 182 msec 2024-11-08T19:13:26,479 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/a8e891c0073a08913834761f455a77e8 2024-11-08T19:13:26,480 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for a8e891c0073a08913834761f455a77e8 2024-11-08T19:13:26,480 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for a8e891c0073a08913834761f455a77e8 2024-11-08T19:13:26,481 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=245339f649708a4ea812249dd3f822c4, ASSIGN in 372 msec 2024-11-08T19:13:26,483 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for a8e891c0073a08913834761f455a77e8 2024-11-08T19:13:26,486 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/a8e891c0073a08913834761f455a77e8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T19:13:26,487 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened a8e891c0073a08913834761f455a77e8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71348022, jitterRate=0.06316837668418884}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T19:13:26,487 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a8e891c0073a08913834761f455a77e8 2024-11-08T19:13:26,487 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for a8e891c0073a08913834761f455a77e8: Running coprocessor pre-open hook at 1731093206470Writing region info on filesystem at 1731093206470Initializing all the Stores at 1731093206473 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731093206473Cleaning up temporary data from old regions at 1731093206480 (+7 ms)Running coprocessor post-open hooks at 1731093206487 (+7 ms)Region opened successfully at 1731093206487 2024-11-08T19:13:26,488 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1731093205731.a8e891c0073a08913834761f455a77e8., pid=34, masterSystemTime=1731093206439 2024-11-08T19:13:26,492 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1731093205731.a8e891c0073a08913834761f455a77e8. 2024-11-08T19:13:26,492 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,ddd,1731093205731.a8e891c0073a08913834761f455a77e8. 2024-11-08T19:13:26,492 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,ggg,1731093205731.ab92690881683c86166f7f23976b5188. 2024-11-08T19:13:26,492 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => ab92690881683c86166f7f23976b5188, NAME => 'test,ggg,1731093205731.ab92690881683c86166f7f23976b5188.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-08T19:13:26,493 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=a8e891c0073a08913834761f455a77e8, regionState=OPEN, openSeqNum=2, regionLocation=692b295ad45c,42693,1731093202977 2024-11-08T19:13:26,493 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ab92690881683c86166f7f23976b5188 2024-11-08T19:13:26,493 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,ggg,1731093205731.ab92690881683c86166f7f23976b5188.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:26,493 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for ab92690881683c86166f7f23976b5188 2024-11-08T19:13:26,493 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for ab92690881683c86166f7f23976b5188 2024-11-08T19:13:26,497 INFO [StoreOpener-ab92690881683c86166f7f23976b5188-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ab92690881683c86166f7f23976b5188 2024-11-08T19:13:26,499 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure a8e891c0073a08913834761f455a77e8, server=692b295ad45c,42693,1731093202977 because future has completed 2024-11-08T19:13:26,500 INFO [StoreOpener-ab92690881683c86166f7f23976b5188-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab92690881683c86166f7f23976b5188 columnFamilyName family 2024-11-08T19:13:26,500 DEBUG [StoreOpener-ab92690881683c86166f7f23976b5188-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:26,501 INFO [StoreOpener-ab92690881683c86166f7f23976b5188-1 {}] regionserver.HStore(327): Store=ab92690881683c86166f7f23976b5188/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T19:13:26,502 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for ab92690881683c86166f7f23976b5188 2024-11-08T19:13:26,503 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/ab92690881683c86166f7f23976b5188 2024-11-08T19:13:26,503 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/ab92690881683c86166f7f23976b5188 2024-11-08T19:13:26,504 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for ab92690881683c86166f7f23976b5188 2024-11-08T19:13:26,504 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for ab92690881683c86166f7f23976b5188 2024-11-08T19:13:26,507 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=8 2024-11-08T19:13:26,507 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure a8e891c0073a08913834761f455a77e8, server=692b295ad45c,42693,1731093202977 in 193 msec 2024-11-08T19:13:26,507 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for ab92690881683c86166f7f23976b5188 2024-11-08T19:13:26,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a8e891c0073a08913834761f455a77e8, ASSIGN in 400 msec 2024-11-08T19:13:26,519 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,uuu,1731093205731.234c6774a52fc6a092eb269a4a390dc7. 2024-11-08T19:13:26,520 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/ab92690881683c86166f7f23976b5188/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T19:13:26,520 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => 234c6774a52fc6a092eb269a4a390dc7, NAME => 'test,uuu,1731093205731.234c6774a52fc6a092eb269a4a390dc7.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-08T19:13:26,520 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 234c6774a52fc6a092eb269a4a390dc7 2024-11-08T19:13:26,520 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,uuu,1731093205731.234c6774a52fc6a092eb269a4a390dc7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:26,520 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened ab92690881683c86166f7f23976b5188; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59191334, jitterRate=-0.11798039078712463}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T19:13:26,520 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for 234c6774a52fc6a092eb269a4a390dc7 2024-11-08T19:13:26,520 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ab92690881683c86166f7f23976b5188 2024-11-08T19:13:26,520 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for 234c6774a52fc6a092eb269a4a390dc7 2024-11-08T19:13:26,521 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for ab92690881683c86166f7f23976b5188: Running coprocessor pre-open hook at 1731093206493Writing region info on filesystem at 1731093206493Initializing all the Stores at 1731093206496 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731093206496Cleaning up temporary data from old regions at 1731093206504 (+8 ms)Running coprocessor post-open hooks at 1731093206520 (+16 ms)Region opened successfully at 1731093206520 2024-11-08T19:13:26,522 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1731093205731.ab92690881683c86166f7f23976b5188., pid=35, masterSystemTime=1731093206439 2024-11-08T19:13:26,524 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1731093205731.ab92690881683c86166f7f23976b5188. 2024-11-08T19:13:26,524 INFO [StoreOpener-234c6774a52fc6a092eb269a4a390dc7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 234c6774a52fc6a092eb269a4a390dc7 2024-11-08T19:13:26,524 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,ggg,1731093205731.ab92690881683c86166f7f23976b5188. 2024-11-08T19:13:26,525 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,vvv,1731093205731.f7c9c940725de9fafd49c71f7bebc530. 2024-11-08T19:13:26,525 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => f7c9c940725de9fafd49c71f7bebc530, NAME => 'test,vvv,1731093205731.f7c9c940725de9fafd49c71f7bebc530.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-08T19:13:26,525 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f7c9c940725de9fafd49c71f7bebc530 2024-11-08T19:13:26,525 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,vvv,1731093205731.f7c9c940725de9fafd49c71f7bebc530.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:26,526 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for f7c9c940725de9fafd49c71f7bebc530 2024-11-08T19:13:26,526 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for f7c9c940725de9fafd49c71f7bebc530 2024-11-08T19:13:26,526 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=ab92690881683c86166f7f23976b5188, regionState=OPEN, openSeqNum=2, regionLocation=692b295ad45c,42693,1731093202977 2024-11-08T19:13:26,529 INFO [StoreOpener-234c6774a52fc6a092eb269a4a390dc7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 234c6774a52fc6a092eb269a4a390dc7 columnFamilyName family 2024-11-08T19:13:26,529 DEBUG [StoreOpener-234c6774a52fc6a092eb269a4a390dc7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:26,530 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab92690881683c86166f7f23976b5188, server=692b295ad45c,42693,1731093202977 because future has completed 2024-11-08T19:13:26,530 INFO [StoreOpener-234c6774a52fc6a092eb269a4a390dc7-1 {}] regionserver.HStore(327): Store=234c6774a52fc6a092eb269a4a390dc7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T19:13:26,530 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for 234c6774a52fc6a092eb269a4a390dc7 2024-11-08T19:13:26,532 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/234c6774a52fc6a092eb269a4a390dc7 2024-11-08T19:13:26,532 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/234c6774a52fc6a092eb269a4a390dc7 2024-11-08T19:13:26,533 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for 234c6774a52fc6a092eb269a4a390dc7 2024-11-08T19:13:26,533 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for 234c6774a52fc6a092eb269a4a390dc7 2024-11-08T19:13:26,535 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=11 2024-11-08T19:13:26,535 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure ab92690881683c86166f7f23976b5188, server=692b295ad45c,42693,1731093202977 in 216 msec 2024-11-08T19:13:26,536 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for 234c6774a52fc6a092eb269a4a390dc7 2024-11-08T19:13:26,538 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ab92690881683c86166f7f23976b5188, ASSIGN in 428 msec 2024-11-08T19:13:26,540 INFO [StoreOpener-f7c9c940725de9fafd49c71f7bebc530-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f7c9c940725de9fafd49c71f7bebc530 2024-11-08T19:13:26,542 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/234c6774a52fc6a092eb269a4a390dc7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T19:13:26,543 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened 234c6774a52fc6a092eb269a4a390dc7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59019101, jitterRate=-0.1205468624830246}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T19:13:26,543 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 234c6774a52fc6a092eb269a4a390dc7 2024-11-08T19:13:26,543 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for 234c6774a52fc6a092eb269a4a390dc7: Running coprocessor pre-open hook at 1731093206521Writing region info on filesystem at 1731093206521Initializing all the Stores at 1731093206522 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731093206522Cleaning up temporary data from old regions at 1731093206533 (+11 ms)Running coprocessor post-open hooks at 1731093206543 (+10 ms)Region opened successfully at 1731093206543 2024-11-08T19:13:26,544 INFO [StoreOpener-f7c9c940725de9fafd49c71f7bebc530-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7c9c940725de9fafd49c71f7bebc530 columnFamilyName family 2024-11-08T19:13:26,544 DEBUG [StoreOpener-f7c9c940725de9fafd49c71f7bebc530-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:26,544 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1731093205731.234c6774a52fc6a092eb269a4a390dc7., pid=47, masterSystemTime=1731093206512 2024-11-08T19:13:26,545 INFO [StoreOpener-f7c9c940725de9fafd49c71f7bebc530-1 {}] regionserver.HStore(327): Store=f7c9c940725de9fafd49c71f7bebc530/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T19:13:26,545 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for f7c9c940725de9fafd49c71f7bebc530 2024-11-08T19:13:26,546 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T19:13:26,548 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/f7c9c940725de9fafd49c71f7bebc530 2024-11-08T19:13:26,548 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50955, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T19:13:26,548 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1731093205731.234c6774a52fc6a092eb269a4a390dc7. 2024-11-08T19:13:26,548 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,uuu,1731093205731.234c6774a52fc6a092eb269a4a390dc7. 2024-11-08T19:13:26,549 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,bbb,1731093205731.8cb2ac5239e07e076f513604c29c2ca8. 2024-11-08T19:13:26,549 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 8cb2ac5239e07e076f513604c29c2ca8, NAME => 'test,bbb,1731093205731.8cb2ac5239e07e076f513604c29c2ca8.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-08T19:13:26,549 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/f7c9c940725de9fafd49c71f7bebc530 2024-11-08T19:13:26,549 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8cb2ac5239e07e076f513604c29c2ca8 2024-11-08T19:13:26,549 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,bbb,1731093205731.8cb2ac5239e07e076f513604c29c2ca8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:26,550 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 8cb2ac5239e07e076f513604c29c2ca8 2024-11-08T19:13:26,550 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 8cb2ac5239e07e076f513604c29c2ca8 2024-11-08T19:13:26,550 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=234c6774a52fc6a092eb269a4a390dc7, regionState=OPEN, openSeqNum=2, regionLocation=692b295ad45c,39647,1731093203061 2024-11-08T19:13:26,550 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for f7c9c940725de9fafd49c71f7bebc530 2024-11-08T19:13:26,550 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for f7c9c940725de9fafd49c71f7bebc530 2024-11-08T19:13:26,555 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 234c6774a52fc6a092eb269a4a390dc7, server=692b295ad45c,39647,1731093203061 because future has completed 2024-11-08T19:13:26,558 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for f7c9c940725de9fafd49c71f7bebc530 2024-11-08T19:13:26,564 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=25 2024-11-08T19:13:26,564 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 234c6774a52fc6a092eb269a4a390dc7, server=692b295ad45c,39647,1731093203061 in 175 msec 2024-11-08T19:13:26,564 INFO [StoreOpener-8cb2ac5239e07e076f513604c29c2ca8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8cb2ac5239e07e076f513604c29c2ca8 2024-11-08T19:13:26,566 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,,1731093205731.d2ea8472f395c484d4998844c4dcd304. 2024-11-08T19:13:26,566 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => d2ea8472f395c484d4998844c4dcd304, NAME => 'test,,1731093205731.d2ea8472f395c484d4998844c4dcd304.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-08T19:13:26,567 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d2ea8472f395c484d4998844c4dcd304 2024-11-08T19:13:26,567 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,,1731093205731.d2ea8472f395c484d4998844c4dcd304.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:26,567 INFO [StoreOpener-8cb2ac5239e07e076f513604c29c2ca8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8cb2ac5239e07e076f513604c29c2ca8 columnFamilyName family 2024-11-08T19:13:26,567 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for d2ea8472f395c484d4998844c4dcd304 2024-11-08T19:13:26,567 DEBUG [StoreOpener-8cb2ac5239e07e076f513604c29c2ca8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:26,567 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for d2ea8472f395c484d4998844c4dcd304 2024-11-08T19:13:26,569 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN in 457 msec 2024-11-08T19:13:26,569 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/f7c9c940725de9fafd49c71f7bebc530/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T19:13:26,568 INFO [StoreOpener-8cb2ac5239e07e076f513604c29c2ca8-1 {}] regionserver.HStore(327): Store=8cb2ac5239e07e076f513604c29c2ca8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T19:13:26,569 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 8cb2ac5239e07e076f513604c29c2ca8 2024-11-08T19:13:26,570 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened f7c9c940725de9fafd49c71f7bebc530; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75451503, jitterRate=0.12431500852108002}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T19:13:26,570 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f7c9c940725de9fafd49c71f7bebc530 2024-11-08T19:13:26,570 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for f7c9c940725de9fafd49c71f7bebc530: Running coprocessor pre-open hook at 1731093206526Writing region info on filesystem at 1731093206526Initializing all the Stores at 1731093206529 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731093206530 (+1 ms)Cleaning up temporary data from old regions at 1731093206550 (+20 ms)Running coprocessor post-open hooks at 1731093206570 (+20 ms)Region opened successfully at 1731093206570 2024-11-08T19:13:26,570 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/8cb2ac5239e07e076f513604c29c2ca8 2024-11-08T19:13:26,571 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/8cb2ac5239e07e076f513604c29c2ca8 2024-11-08T19:13:26,572 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1731093205731.f7c9c940725de9fafd49c71f7bebc530., pid=39, masterSystemTime=1731093206439 2024-11-08T19:13:26,572 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 8cb2ac5239e07e076f513604c29c2ca8 2024-11-08T19:13:26,572 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 8cb2ac5239e07e076f513604c29c2ca8 2024-11-08T19:13:26,575 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1731093205731.f7c9c940725de9fafd49c71f7bebc530. 2024-11-08T19:13:26,575 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,vvv,1731093205731.f7c9c940725de9fafd49c71f7bebc530. 2024-11-08T19:13:26,575 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,sss,1731093205731.77e3781e625f0b456041758476a7d6b2. 2024-11-08T19:13:26,575 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 77e3781e625f0b456041758476a7d6b2, NAME => 'test,sss,1731093205731.77e3781e625f0b456041758476a7d6b2.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-08T19:13:26,576 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 77e3781e625f0b456041758476a7d6b2 2024-11-08T19:13:26,576 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,sss,1731093205731.77e3781e625f0b456041758476a7d6b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:26,577 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 77e3781e625f0b456041758476a7d6b2 2024-11-08T19:13:26,577 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 77e3781e625f0b456041758476a7d6b2 2024-11-08T19:13:26,577 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=f7c9c940725de9fafd49c71f7bebc530, regionState=OPEN, openSeqNum=2, regionLocation=692b295ad45c,42693,1731093202977 2024-11-08T19:13:26,579 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 8cb2ac5239e07e076f513604c29c2ca8 2024-11-08T19:13:26,580 INFO [StoreOpener-d2ea8472f395c484d4998844c4dcd304-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d2ea8472f395c484d4998844c4dcd304 2024-11-08T19:13:26,580 INFO [StoreOpener-77e3781e625f0b456041758476a7d6b2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 77e3781e625f0b456041758476a7d6b2 2024-11-08T19:13:26,581 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure f7c9c940725de9fafd49c71f7bebc530, server=692b295ad45c,42693,1731093202977 because future has completed 2024-11-08T19:13:26,583 INFO [StoreOpener-77e3781e625f0b456041758476a7d6b2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 77e3781e625f0b456041758476a7d6b2 columnFamilyName family 2024-11-08T19:13:26,584 DEBUG [StoreOpener-77e3781e625f0b456041758476a7d6b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:26,585 INFO [StoreOpener-d2ea8472f395c484d4998844c4dcd304-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2ea8472f395c484d4998844c4dcd304 columnFamilyName family 2024-11-08T19:13:26,586 DEBUG [StoreOpener-d2ea8472f395c484d4998844c4dcd304-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:26,586 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/8cb2ac5239e07e076f513604c29c2ca8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T19:13:26,586 INFO [StoreOpener-77e3781e625f0b456041758476a7d6b2-1 {}] regionserver.HStore(327): Store=77e3781e625f0b456041758476a7d6b2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T19:13:26,586 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 77e3781e625f0b456041758476a7d6b2 2024-11-08T19:13:26,586 INFO [StoreOpener-d2ea8472f395c484d4998844c4dcd304-1 {}] regionserver.HStore(327): Store=d2ea8472f395c484d4998844c4dcd304/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T19:13:26,587 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for d2ea8472f395c484d4998844c4dcd304 2024-11-08T19:13:26,587 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/77e3781e625f0b456041758476a7d6b2 2024-11-08T19:13:26,588 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=26 2024-11-08T19:13:26,588 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure f7c9c940725de9fafd49c71f7bebc530, server=692b295ad45c,42693,1731093202977 in 246 msec 2024-11-08T19:13:26,588 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/77e3781e625f0b456041758476a7d6b2 2024-11-08T19:13:26,588 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 8cb2ac5239e07e076f513604c29c2ca8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60140585, jitterRate=-0.10383544862270355}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T19:13:26,589 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 77e3781e625f0b456041758476a7d6b2 2024-11-08T19:13:26,589 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/d2ea8472f395c484d4998844c4dcd304 2024-11-08T19:13:26,589 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 77e3781e625f0b456041758476a7d6b2 2024-11-08T19:13:26,588 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8cb2ac5239e07e076f513604c29c2ca8 2024-11-08T19:13:26,590 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 8cb2ac5239e07e076f513604c29c2ca8: Running coprocessor pre-open hook at 1731093206550Writing region info on filesystem at 1731093206550Initializing all the Stores at 1731093206554 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731093206554Cleaning up temporary data from old regions at 1731093206572 (+18 ms)Running coprocessor post-open hooks at 1731093206590 (+18 ms)Region opened successfully at 1731093206590 2024-11-08T19:13:26,590 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/d2ea8472f395c484d4998844c4dcd304 2024-11-08T19:13:26,591 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for d2ea8472f395c484d4998844c4dcd304 2024-11-08T19:13:26,591 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1731093205731.8cb2ac5239e07e076f513604c29c2ca8., pid=42, masterSystemTime=1731093206512 2024-11-08T19:13:26,591 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for d2ea8472f395c484d4998844c4dcd304 2024-11-08T19:13:26,592 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f7c9c940725de9fafd49c71f7bebc530, ASSIGN in 481 msec 2024-11-08T19:13:26,593 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 77e3781e625f0b456041758476a7d6b2 2024-11-08T19:13:26,595 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for d2ea8472f395c484d4998844c4dcd304 2024-11-08T19:13:26,595 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1731093205731.8cb2ac5239e07e076f513604c29c2ca8. 2024-11-08T19:13:26,595 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,bbb,1731093205731.8cb2ac5239e07e076f513604c29c2ca8. 2024-11-08T19:13:26,596 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,yyy,1731093205731.22cccf3cc4c2d3343ceae07ff16f899f. 2024-11-08T19:13:26,596 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 22cccf3cc4c2d3343ceae07ff16f899f, NAME => 'test,yyy,1731093205731.22cccf3cc4c2d3343ceae07ff16f899f.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-08T19:13:26,596 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 22cccf3cc4c2d3343ceae07ff16f899f 2024-11-08T19:13:26,597 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,yyy,1731093205731.22cccf3cc4c2d3343ceae07ff16f899f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:26,597 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 22cccf3cc4c2d3343ceae07ff16f899f 2024-11-08T19:13:26,597 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 22cccf3cc4c2d3343ceae07ff16f899f 2024-11-08T19:13:26,597 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=8cb2ac5239e07e076f513604c29c2ca8, regionState=OPEN, openSeqNum=2, regionLocation=692b295ad45c,39647,1731093203061 2024-11-08T19:13:26,601 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,39647,1731093203061 because future has completed 2024-11-08T19:13:26,605 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/77e3781e625f0b456041758476a7d6b2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T19:13:26,606 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 77e3781e625f0b456041758476a7d6b2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61337262, jitterRate=-0.08600357174873352}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T19:13:26,607 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 77e3781e625f0b456041758476a7d6b2 2024-11-08T19:13:26,607 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 77e3781e625f0b456041758476a7d6b2: Running coprocessor pre-open hook at 1731093206577Writing region info on filesystem at 1731093206577Initializing all the Stores at 1731093206579 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731093206579Cleaning up temporary data from old regions at 1731093206589 (+10 ms)Running coprocessor post-open hooks at 1731093206607 (+18 ms)Region opened successfully at 1731093206607 2024-11-08T19:13:26,608 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1731093205731.77e3781e625f0b456041758476a7d6b2., pid=38, masterSystemTime=1731093206439 2024-11-08T19:13:26,608 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=6 2024-11-08T19:13:26,608 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,39647,1731093203061 in 242 msec 2024-11-08T19:13:26,611 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1731093205731.77e3781e625f0b456041758476a7d6b2. 2024-11-08T19:13:26,611 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,sss,1731093205731.77e3781e625f0b456041758476a7d6b2. 2024-11-08T19:13:26,611 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,xxx,1731093205731.48bb25f64ce0dca8172688cf9a77f57b. 2024-11-08T19:13:26,611 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 48bb25f64ce0dca8172688cf9a77f57b, NAME => 'test,xxx,1731093205731.48bb25f64ce0dca8172688cf9a77f57b.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-08T19:13:26,611 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN in 501 msec 2024-11-08T19:13:26,612 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 48bb25f64ce0dca8172688cf9a77f57b 2024-11-08T19:13:26,612 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,xxx,1731093205731.48bb25f64ce0dca8172688cf9a77f57b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:26,612 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 48bb25f64ce0dca8172688cf9a77f57b 2024-11-08T19:13:26,612 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 48bb25f64ce0dca8172688cf9a77f57b 2024-11-08T19:13:26,612 INFO [StoreOpener-22cccf3cc4c2d3343ceae07ff16f899f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 22cccf3cc4c2d3343ceae07ff16f899f 2024-11-08T19:13:26,614 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=77e3781e625f0b456041758476a7d6b2, regionState=OPEN, openSeqNum=2, regionLocation=692b295ad45c,42693,1731093202977 2024-11-08T19:13:26,615 INFO [StoreOpener-22cccf3cc4c2d3343ceae07ff16f899f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22cccf3cc4c2d3343ceae07ff16f899f columnFamilyName family 2024-11-08T19:13:26,615 DEBUG [StoreOpener-22cccf3cc4c2d3343ceae07ff16f899f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:26,616 INFO [StoreOpener-22cccf3cc4c2d3343ceae07ff16f899f-1 {}] regionserver.HStore(327): Store=22cccf3cc4c2d3343ceae07ff16f899f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T19:13:26,616 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 22cccf3cc4c2d3343ceae07ff16f899f 2024-11-08T19:13:26,617 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 77e3781e625f0b456041758476a7d6b2, server=692b295ad45c,42693,1731093202977 because future has completed 2024-11-08T19:13:26,618 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/22cccf3cc4c2d3343ceae07ff16f899f 2024-11-08T19:13:26,619 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/22cccf3cc4c2d3343ceae07ff16f899f 2024-11-08T19:13:26,620 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 22cccf3cc4c2d3343ceae07ff16f899f 2024-11-08T19:13:26,620 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 22cccf3cc4c2d3343ceae07ff16f899f 2024-11-08T19:13:26,623 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 22cccf3cc4c2d3343ceae07ff16f899f 2024-11-08T19:13:26,624 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=23 2024-11-08T19:13:26,624 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 77e3781e625f0b456041758476a7d6b2, server=692b295ad45c,42693,1731093202977 in 288 msec 2024-11-08T19:13:26,629 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/22cccf3cc4c2d3343ceae07ff16f899f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T19:13:26,630 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=77e3781e625f0b456041758476a7d6b2, ASSIGN in 518 msec 2024-11-08T19:13:26,630 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 22cccf3cc4c2d3343ceae07ff16f899f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74128522, jitterRate=0.10460105538368225}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T19:13:26,631 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 22cccf3cc4c2d3343ceae07ff16f899f 2024-11-08T19:13:26,631 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 22cccf3cc4c2d3343ceae07ff16f899f: Running coprocessor pre-open hook at 1731093206597Writing region info on filesystem at 1731093206597Initializing all the Stores at 1731093206599 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731093206599Cleaning up temporary data from old regions at 1731093206620 (+21 ms)Running coprocessor post-open hooks at 1731093206631 (+11 ms)Region opened successfully at 1731093206631 2024-11-08T19:13:26,632 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/d2ea8472f395c484d4998844c4dcd304/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T19:13:26,633 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened d2ea8472f395c484d4998844c4dcd304; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60606009, jitterRate=-0.0969000905752182}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T19:13:26,633 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d2ea8472f395c484d4998844c4dcd304 2024-11-08T19:13:26,633 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for d2ea8472f395c484d4998844c4dcd304: Running coprocessor pre-open hook at 1731093206567Writing region info on filesystem at 1731093206567Initializing all the Stores at 1731093206569 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731093206569Cleaning up temporary data from old regions at 1731093206591 (+22 ms)Running coprocessor post-open hooks at 1731093206633 (+42 ms)Region opened successfully at 1731093206633 2024-11-08T19:13:26,634 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1731093205731.22cccf3cc4c2d3343ceae07ff16f899f., pid=40, masterSystemTime=1731093206512 2024-11-08T19:13:26,635 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1731093205731.d2ea8472f395c484d4998844c4dcd304., pid=55, masterSystemTime=1731093206545 2024-11-08T19:13:26,638 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1731093205731.22cccf3cc4c2d3343ceae07ff16f899f. 2024-11-08T19:13:26,638 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=22cccf3cc4c2d3343ceae07ff16f899f, regionState=OPEN, openSeqNum=2, regionLocation=692b295ad45c,39647,1731093203061 2024-11-08T19:13:26,638 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,yyy,1731093205731.22cccf3cc4c2d3343ceae07ff16f899f. 2024-11-08T19:13:26,638 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1731093205731.d2ea8472f395c484d4998844c4dcd304. 2024-11-08T19:13:26,638 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,,1731093205731.d2ea8472f395c484d4998844c4dcd304. 2024-11-08T19:13:26,639 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,rrr,1731093205731.f1496a2d5d6037cbe8d2f7a8f1407184. 2024-11-08T19:13:26,639 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,nnn,1731093205731.07363d44bad100b56e1b1aba358b0ed2. 2024-11-08T19:13:26,639 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => f1496a2d5d6037cbe8d2f7a8f1407184, NAME => 'test,rrr,1731093205731.f1496a2d5d6037cbe8d2f7a8f1407184.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-08T19:13:26,639 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 07363d44bad100b56e1b1aba358b0ed2, NAME => 'test,nnn,1731093205731.07363d44bad100b56e1b1aba358b0ed2.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-08T19:13:26,639 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f1496a2d5d6037cbe8d2f7a8f1407184 2024-11-08T19:13:26,639 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 07363d44bad100b56e1b1aba358b0ed2 2024-11-08T19:13:26,639 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,nnn,1731093205731.07363d44bad100b56e1b1aba358b0ed2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:26,639 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,rrr,1731093205731.f1496a2d5d6037cbe8d2f7a8f1407184.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:26,639 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 07363d44bad100b56e1b1aba358b0ed2 2024-11-08T19:13:26,640 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for f1496a2d5d6037cbe8d2f7a8f1407184 2024-11-08T19:13:26,640 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 07363d44bad100b56e1b1aba358b0ed2 2024-11-08T19:13:26,640 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for f1496a2d5d6037cbe8d2f7a8f1407184 2024-11-08T19:13:26,640 INFO [StoreOpener-48bb25f64ce0dca8172688cf9a77f57b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 48bb25f64ce0dca8172688cf9a77f57b 2024-11-08T19:13:26,643 INFO [StoreOpener-f1496a2d5d6037cbe8d2f7a8f1407184-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f1496a2d5d6037cbe8d2f7a8f1407184 2024-11-08T19:13:26,644 INFO [StoreOpener-48bb25f64ce0dca8172688cf9a77f57b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 48bb25f64ce0dca8172688cf9a77f57b columnFamilyName family 2024-11-08T19:13:26,644 DEBUG [StoreOpener-48bb25f64ce0dca8172688cf9a77f57b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:26,646 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22cccf3cc4c2d3343ceae07ff16f899f, server=692b295ad45c,39647,1731093203061 because future has completed 2024-11-08T19:13:26,646 INFO [StoreOpener-48bb25f64ce0dca8172688cf9a77f57b-1 {}] regionserver.HStore(327): Store=48bb25f64ce0dca8172688cf9a77f57b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T19:13:26,644 INFO [StoreOpener-07363d44bad100b56e1b1aba358b0ed2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 07363d44bad100b56e1b1aba358b0ed2 2024-11-08T19:13:26,646 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 48bb25f64ce0dca8172688cf9a77f57b 2024-11-08T19:13:26,647 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/48bb25f64ce0dca8172688cf9a77f57b 2024-11-08T19:13:26,648 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d2ea8472f395c484d4998844c4dcd304, regionState=OPEN, openSeqNum=2, regionLocation=692b295ad45c,36007,1731093202744 2024-11-08T19:13:26,648 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/48bb25f64ce0dca8172688cf9a77f57b 2024-11-08T19:13:26,649 INFO [StoreOpener-f1496a2d5d6037cbe8d2f7a8f1407184-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1496a2d5d6037cbe8d2f7a8f1407184 columnFamilyName family 2024-11-08T19:13:26,650 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39509 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=692b295ad45c,39647,1731093203061, table=test, region=22cccf3cc4c2d3343ceae07ff16f899f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-08T19:13:26,650 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 48bb25f64ce0dca8172688cf9a77f57b 2024-11-08T19:13:26,650 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 48bb25f64ce0dca8172688cf9a77f57b 2024-11-08T19:13:26,651 DEBUG [StoreOpener-f1496a2d5d6037cbe8d2f7a8f1407184-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:26,651 INFO [StoreOpener-f1496a2d5d6037cbe8d2f7a8f1407184-1 {}] regionserver.HStore(327): Store=f1496a2d5d6037cbe8d2f7a8f1407184/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T19:13:26,652 INFO [StoreOpener-07363d44bad100b56e1b1aba358b0ed2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 07363d44bad100b56e1b1aba358b0ed2 columnFamilyName family 2024-11-08T19:13:26,652 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for f1496a2d5d6037cbe8d2f7a8f1407184 2024-11-08T19:13:26,653 DEBUG [StoreOpener-07363d44bad100b56e1b1aba358b0ed2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:26,654 INFO [StoreOpener-07363d44bad100b56e1b1aba358b0ed2-1 {}] regionserver.HStore(327): Store=07363d44bad100b56e1b1aba358b0ed2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T19:13:26,654 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 07363d44bad100b56e1b1aba358b0ed2 2024-11-08T19:13:26,654 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/f1496a2d5d6037cbe8d2f7a8f1407184 2024-11-08T19:13:26,654 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2ea8472f395c484d4998844c4dcd304, server=692b295ad45c,36007,1731093202744 because future has completed 2024-11-08T19:13:26,655 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/f1496a2d5d6037cbe8d2f7a8f1407184 2024-11-08T19:13:26,655 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 48bb25f64ce0dca8172688cf9a77f57b 2024-11-08T19:13:26,655 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/07363d44bad100b56e1b1aba358b0ed2 2024-11-08T19:13:26,655 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for f1496a2d5d6037cbe8d2f7a8f1407184 2024-11-08T19:13:26,656 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for f1496a2d5d6037cbe8d2f7a8f1407184 2024-11-08T19:13:26,656 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/07363d44bad100b56e1b1aba358b0ed2 2024-11-08T19:13:26,657 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 07363d44bad100b56e1b1aba358b0ed2 2024-11-08T19:13:26,657 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 07363d44bad100b56e1b1aba358b0ed2 2024-11-08T19:13:26,659 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for f1496a2d5d6037cbe8d2f7a8f1407184 2024-11-08T19:13:26,660 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 07363d44bad100b56e1b1aba358b0ed2 2024-11-08T19:13:26,664 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=29 2024-11-08T19:13:26,664 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 22cccf3cc4c2d3343ceae07ff16f899f, server=692b295ad45c,39647,1731093203061 in 308 msec 2024-11-08T19:13:26,667 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN in 558 msec 2024-11-08T19:13:26,669 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/48bb25f64ce0dca8172688cf9a77f57b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T19:13:26,670 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 48bb25f64ce0dca8172688cf9a77f57b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66518435, jitterRate=-0.008798077702522278}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T19:13:26,670 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 48bb25f64ce0dca8172688cf9a77f57b 2024-11-08T19:13:26,670 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 48bb25f64ce0dca8172688cf9a77f57b: Running coprocessor pre-open hook at 1731093206612Writing region info on filesystem at 1731093206612Initializing all the Stores at 1731093206614 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731093206614Cleaning up temporary data from old regions at 1731093206650 (+36 ms)Running coprocessor post-open hooks at 1731093206670 (+20 ms)Region opened successfully at 1731093206670 2024-11-08T19:13:26,671 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1731093205731.48bb25f64ce0dca8172688cf9a77f57b., pid=31, masterSystemTime=1731093206439 2024-11-08T19:13:26,674 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1731093205731.48bb25f64ce0dca8172688cf9a77f57b. 2024-11-08T19:13:26,674 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,xxx,1731093205731.48bb25f64ce0dca8172688cf9a77f57b. 2024-11-08T19:13:26,674 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,ccc,1731093205731.7b4a23d5efe8e832c0d5cb16df3b1369. 2024-11-08T19:13:26,674 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 7b4a23d5efe8e832c0d5cb16df3b1369, NAME => 'test,ccc,1731093205731.7b4a23d5efe8e832c0d5cb16df3b1369.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-08T19:13:26,675 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7b4a23d5efe8e832c0d5cb16df3b1369 2024-11-08T19:13:26,675 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,ccc,1731093205731.7b4a23d5efe8e832c0d5cb16df3b1369.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:26,675 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 7b4a23d5efe8e832c0d5cb16df3b1369 2024-11-08T19:13:26,675 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 7b4a23d5efe8e832c0d5cb16df3b1369 2024-11-08T19:13:26,677 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=5 2024-11-08T19:13:26,677 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure d2ea8472f395c484d4998844c4dcd304, server=692b295ad45c,36007,1731093202744 in 241 msec 2024-11-08T19:13:26,680 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/f1496a2d5d6037cbe8d2f7a8f1407184/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T19:13:26,681 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened f1496a2d5d6037cbe8d2f7a8f1407184; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62373655, jitterRate=-0.07056011259555817}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T19:13:26,682 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f1496a2d5d6037cbe8d2f7a8f1407184 2024-11-08T19:13:26,682 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d2ea8472f395c484d4998844c4dcd304, ASSIGN in 571 msec 2024-11-08T19:13:26,682 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for f1496a2d5d6037cbe8d2f7a8f1407184: Running coprocessor pre-open hook at 1731093206640Writing region info on filesystem at 1731093206640Initializing all the Stores at 1731093206642 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731093206642Cleaning up temporary data from old regions at 1731093206656 (+14 ms)Running coprocessor post-open hooks at 1731093206682 (+26 ms)Region opened successfully at 1731093206682 2024-11-08T19:13:26,683 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1731093205731.f1496a2d5d6037cbe8d2f7a8f1407184., pid=46, masterSystemTime=1731093206512 2024-11-08T19:13:26,684 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=48bb25f64ce0dca8172688cf9a77f57b, regionState=OPEN, openSeqNum=2, regionLocation=692b295ad45c,42693,1731093202977 2024-11-08T19:13:26,686 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1731093205731.f1496a2d5d6037cbe8d2f7a8f1407184. 2024-11-08T19:13:26,687 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,rrr,1731093205731.f1496a2d5d6037cbe8d2f7a8f1407184. 2024-11-08T19:13:26,687 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,fff,1731093205731.53b7061bfce2939f050c57221eed7f2a. 2024-11-08T19:13:26,687 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 53b7061bfce2939f050c57221eed7f2a, NAME => 'test,fff,1731093205731.53b7061bfce2939f050c57221eed7f2a.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-08T19:13:26,688 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 53b7061bfce2939f050c57221eed7f2a 2024-11-08T19:13:26,688 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 48bb25f64ce0dca8172688cf9a77f57b, server=692b295ad45c,42693,1731093202977 because future has completed 2024-11-08T19:13:26,688 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,fff,1731093205731.53b7061bfce2939f050c57221eed7f2a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:26,689 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/07363d44bad100b56e1b1aba358b0ed2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T19:13:26,689 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 53b7061bfce2939f050c57221eed7f2a 2024-11-08T19:13:26,689 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 53b7061bfce2939f050c57221eed7f2a 2024-11-08T19:13:26,689 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=f1496a2d5d6037cbe8d2f7a8f1407184, regionState=OPEN, openSeqNum=2, regionLocation=692b295ad45c,39647,1731093203061 2024-11-08T19:13:26,690 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 07363d44bad100b56e1b1aba358b0ed2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61045976, jitterRate=-0.09034407138824463}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T19:13:26,690 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 07363d44bad100b56e1b1aba358b0ed2 2024-11-08T19:13:26,690 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 07363d44bad100b56e1b1aba358b0ed2: Running coprocessor pre-open hook at 1731093206640Writing region info on filesystem at 1731093206640Initializing all the Stores at 1731093206642 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731093206642Cleaning up temporary data from old regions at 1731093206657 (+15 ms)Running coprocessor post-open hooks at 1731093206690 (+33 ms)Region opened successfully at 1731093206690 2024-11-08T19:13:26,691 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1731093205731.07363d44bad100b56e1b1aba358b0ed2., pid=51, masterSystemTime=1731093206545 2024-11-08T19:13:26,693 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1496a2d5d6037cbe8d2f7a8f1407184, server=692b295ad45c,39647,1731093203061 because future has completed 2024-11-08T19:13:26,693 INFO [StoreOpener-7b4a23d5efe8e832c0d5cb16df3b1369-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7b4a23d5efe8e832c0d5cb16df3b1369 2024-11-08T19:13:26,694 INFO [StoreOpener-53b7061bfce2939f050c57221eed7f2a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 53b7061bfce2939f050c57221eed7f2a 2024-11-08T19:13:26,695 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1731093205731.07363d44bad100b56e1b1aba358b0ed2. 2024-11-08T19:13:26,695 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,nnn,1731093205731.07363d44bad100b56e1b1aba358b0ed2. 2024-11-08T19:13:26,695 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,qqq,1731093205731.caa4304cf54e018c1a1a2071373865b5. 2024-11-08T19:13:26,696 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => caa4304cf54e018c1a1a2071373865b5, NAME => 'test,qqq,1731093205731.caa4304cf54e018c1a1a2071373865b5.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-08T19:13:26,696 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test caa4304cf54e018c1a1a2071373865b5 2024-11-08T19:13:26,696 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,qqq,1731093205731.caa4304cf54e018c1a1a2071373865b5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:26,696 INFO [StoreOpener-53b7061bfce2939f050c57221eed7f2a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 53b7061bfce2939f050c57221eed7f2a columnFamilyName family 2024-11-08T19:13:26,696 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for caa4304cf54e018c1a1a2071373865b5 2024-11-08T19:13:26,697 INFO [StoreOpener-7b4a23d5efe8e832c0d5cb16df3b1369-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b4a23d5efe8e832c0d5cb16df3b1369 columnFamilyName family 2024-11-08T19:13:26,697 DEBUG [StoreOpener-53b7061bfce2939f050c57221eed7f2a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:26,697 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for caa4304cf54e018c1a1a2071373865b5 2024-11-08T19:13:26,697 DEBUG [StoreOpener-7b4a23d5efe8e832c0d5cb16df3b1369-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:26,697 INFO [StoreOpener-53b7061bfce2939f050c57221eed7f2a-1 {}] regionserver.HStore(327): Store=53b7061bfce2939f050c57221eed7f2a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T19:13:26,697 INFO [StoreOpener-7b4a23d5efe8e832c0d5cb16df3b1369-1 {}] regionserver.HStore(327): Store=7b4a23d5efe8e832c0d5cb16df3b1369/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T19:13:26,697 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 7b4a23d5efe8e832c0d5cb16df3b1369 2024-11-08T19:13:26,698 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 53b7061bfce2939f050c57221eed7f2a 2024-11-08T19:13:26,699 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/7b4a23d5efe8e832c0d5cb16df3b1369 2024-11-08T19:13:26,699 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/7b4a23d5efe8e832c0d5cb16df3b1369 2024-11-08T19:13:26,699 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/53b7061bfce2939f050c57221eed7f2a 2024-11-08T19:13:26,700 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/53b7061bfce2939f050c57221eed7f2a 2024-11-08T19:13:26,700 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 7b4a23d5efe8e832c0d5cb16df3b1369 2024-11-08T19:13:26,700 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 7b4a23d5efe8e832c0d5cb16df3b1369 2024-11-08T19:13:26,700 INFO [StoreOpener-caa4304cf54e018c1a1a2071373865b5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region caa4304cf54e018c1a1a2071373865b5 2024-11-08T19:13:26,701 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 53b7061bfce2939f050c57221eed7f2a 2024-11-08T19:13:26,701 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 53b7061bfce2939f050c57221eed7f2a 2024-11-08T19:13:26,701 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=28 2024-11-08T19:13:26,701 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure 48bb25f64ce0dca8172688cf9a77f57b, server=692b295ad45c,42693,1731093202977 in 412 msec 2024-11-08T19:13:26,703 INFO [StoreOpener-caa4304cf54e018c1a1a2071373865b5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region caa4304cf54e018c1a1a2071373865b5 columnFamilyName family 2024-11-08T19:13:26,703 DEBUG [StoreOpener-caa4304cf54e018c1a1a2071373865b5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:26,704 INFO [StoreOpener-caa4304cf54e018c1a1a2071373865b5-1 {}] regionserver.HStore(327): Store=caa4304cf54e018c1a1a2071373865b5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T19:13:26,704 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for caa4304cf54e018c1a1a2071373865b5 2024-11-08T19:13:26,704 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 7b4a23d5efe8e832c0d5cb16df3b1369 2024-11-08T19:13:26,705 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 53b7061bfce2939f050c57221eed7f2a 2024-11-08T19:13:26,705 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/caa4304cf54e018c1a1a2071373865b5 2024-11-08T19:13:26,706 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/caa4304cf54e018c1a1a2071373865b5 2024-11-08T19:13:26,706 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for caa4304cf54e018c1a1a2071373865b5 2024-11-08T19:13:26,706 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for caa4304cf54e018c1a1a2071373865b5 2024-11-08T19:13:26,708 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/53b7061bfce2939f050c57221eed7f2a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T19:13:26,709 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/7b4a23d5efe8e832c0d5cb16df3b1369/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T19:13:26,709 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 53b7061bfce2939f050c57221eed7f2a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65519375, jitterRate=-0.023685231804847717}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T19:13:26,709 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 7b4a23d5efe8e832c0d5cb16df3b1369; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60062007, jitterRate=-0.10500635206699371}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T19:13:26,710 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7b4a23d5efe8e832c0d5cb16df3b1369 2024-11-08T19:13:26,710 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 53b7061bfce2939f050c57221eed7f2a 2024-11-08T19:13:26,710 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for caa4304cf54e018c1a1a2071373865b5 2024-11-08T19:13:26,710 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 7b4a23d5efe8e832c0d5cb16df3b1369: Running coprocessor pre-open hook at 1731093206675Writing region info on filesystem at 1731093206675Initializing all the Stores at 1731093206677 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731093206678 (+1 ms)Cleaning up temporary data from old regions at 1731093206700 (+22 ms)Running coprocessor post-open hooks at 1731093206710 (+10 ms)Region opened successfully at 1731093206710 2024-11-08T19:13:26,710 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 53b7061bfce2939f050c57221eed7f2a: Running coprocessor pre-open hook at 1731093206689Writing region info on filesystem at 1731093206689Initializing all the Stores at 1731093206691 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731093206691Cleaning up temporary data from old regions at 1731093206701 (+10 ms)Running coprocessor post-open hooks at 1731093206710 (+9 ms)Region opened successfully at 1731093206710 2024-11-08T19:13:26,710 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=48bb25f64ce0dca8172688cf9a77f57b, ASSIGN in 595 msec 2024-11-08T19:13:26,712 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1731093205731.7b4a23d5efe8e832c0d5cb16df3b1369., pid=33, masterSystemTime=1731093206439 2024-11-08T19:13:26,712 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=07363d44bad100b56e1b1aba358b0ed2, regionState=OPEN, openSeqNum=2, regionLocation=692b295ad45c,36007,1731093202744 2024-11-08T19:13:26,712 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1731093205731.53b7061bfce2939f050c57221eed7f2a., pid=43, masterSystemTime=1731093206512 2024-11-08T19:13:26,715 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/caa4304cf54e018c1a1a2071373865b5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T19:13:26,716 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1731093205731.7b4a23d5efe8e832c0d5cb16df3b1369. 2024-11-08T19:13:26,716 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07363d44bad100b56e1b1aba358b0ed2, server=692b295ad45c,36007,1731093202744 because future has completed 2024-11-08T19:13:26,716 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,ccc,1731093205731.7b4a23d5efe8e832c0d5cb16df3b1369. 2024-11-08T19:13:26,716 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,ppp,1731093205731.e92192d7abb7a6c62f8815702d7dc38d. 2024-11-08T19:13:26,716 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened caa4304cf54e018c1a1a2071373865b5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59453275, jitterRate=-0.11407716572284698}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T19:13:26,717 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => e92192d7abb7a6c62f8815702d7dc38d, NAME => 'test,ppp,1731093205731.e92192d7abb7a6c62f8815702d7dc38d.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-08T19:13:26,717 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for caa4304cf54e018c1a1a2071373865b5 2024-11-08T19:13:26,717 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for caa4304cf54e018c1a1a2071373865b5: Running coprocessor pre-open hook at 1731093206697Writing region info on filesystem at 1731093206697Initializing all the Stores at 1731093206699 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731093206699Cleaning up temporary data from old regions at 1731093206706 (+7 ms)Running coprocessor post-open hooks at 1731093206717 (+11 ms)Region opened successfully at 1731093206717 2024-11-08T19:13:26,717 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e92192d7abb7a6c62f8815702d7dc38d 2024-11-08T19:13:26,717 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,ppp,1731093205731.e92192d7abb7a6c62f8815702d7dc38d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:26,717 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for e92192d7abb7a6c62f8815702d7dc38d 2024-11-08T19:13:26,717 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for e92192d7abb7a6c62f8815702d7dc38d 2024-11-08T19:13:26,718 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1731093205731.caa4304cf54e018c1a1a2071373865b5., pid=49, masterSystemTime=1731093206545 2024-11-08T19:13:26,720 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1731093205731.53b7061bfce2939f050c57221eed7f2a. 2024-11-08T19:13:26,720 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,fff,1731093205731.53b7061bfce2939f050c57221eed7f2a. 2024-11-08T19:13:26,720 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,iii,1731093205731.2548b6c044cd2c9ceb7dec340e2cde6c. 2024-11-08T19:13:26,721 INFO [StoreOpener-e92192d7abb7a6c62f8815702d7dc38d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e92192d7abb7a6c62f8815702d7dc38d 2024-11-08T19:13:26,721 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 2548b6c044cd2c9ceb7dec340e2cde6c, NAME => 'test,iii,1731093205731.2548b6c044cd2c9ceb7dec340e2cde6c.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-08T19:13:26,721 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2548b6c044cd2c9ceb7dec340e2cde6c 2024-11-08T19:13:26,722 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,iii,1731093205731.2548b6c044cd2c9ceb7dec340e2cde6c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:26,722 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1731093205731.caa4304cf54e018c1a1a2071373865b5. 2024-11-08T19:13:26,722 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,qqq,1731093205731.caa4304cf54e018c1a1a2071373865b5. 2024-11-08T19:13:26,722 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,www,1731093205731.9e50623b9f9d6b5fc998b958dd629458. 2024-11-08T19:13:26,722 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => 9e50623b9f9d6b5fc998b958dd629458, NAME => 'test,www,1731093205731.9e50623b9f9d6b5fc998b958dd629458.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-08T19:13:26,722 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 2548b6c044cd2c9ceb7dec340e2cde6c 2024-11-08T19:13:26,723 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 2548b6c044cd2c9ceb7dec340e2cde6c 2024-11-08T19:13:26,723 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 9e50623b9f9d6b5fc998b958dd629458 2024-11-08T19:13:26,723 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,www,1731093205731.9e50623b9f9d6b5fc998b958dd629458.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:26,723 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for 9e50623b9f9d6b5fc998b958dd629458 2024-11-08T19:13:26,723 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for 9e50623b9f9d6b5fc998b958dd629458 2024-11-08T19:13:26,724 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=22 2024-11-08T19:13:26,724 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure f1496a2d5d6037cbe8d2f7a8f1407184, server=692b295ad45c,39647,1731093203061 in 339 msec 2024-11-08T19:13:26,724 INFO [StoreOpener-e92192d7abb7a6c62f8815702d7dc38d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e92192d7abb7a6c62f8815702d7dc38d columnFamilyName family 2024-11-08T19:13:26,724 DEBUG [StoreOpener-e92192d7abb7a6c62f8815702d7dc38d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:26,725 INFO [StoreOpener-9e50623b9f9d6b5fc998b958dd629458-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 9e50623b9f9d6b5fc998b958dd629458 2024-11-08T19:13:26,725 INFO [StoreOpener-e92192d7abb7a6c62f8815702d7dc38d-1 {}] regionserver.HStore(327): Store=e92192d7abb7a6c62f8815702d7dc38d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T19:13:26,725 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for e92192d7abb7a6c62f8815702d7dc38d 2024-11-08T19:13:26,726 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/e92192d7abb7a6c62f8815702d7dc38d 2024-11-08T19:13:26,727 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/e92192d7abb7a6c62f8815702d7dc38d 2024-11-08T19:13:26,727 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for e92192d7abb7a6c62f8815702d7dc38d 2024-11-08T19:13:26,727 INFO [StoreOpener-9e50623b9f9d6b5fc998b958dd629458-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9e50623b9f9d6b5fc998b958dd629458 columnFamilyName family 2024-11-08T19:13:26,727 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for e92192d7abb7a6c62f8815702d7dc38d 2024-11-08T19:13:26,727 DEBUG [StoreOpener-9e50623b9f9d6b5fc998b958dd629458-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:26,728 INFO [StoreOpener-9e50623b9f9d6b5fc998b958dd629458-1 {}] regionserver.HStore(327): Store=9e50623b9f9d6b5fc998b958dd629458/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T19:13:26,728 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for 9e50623b9f9d6b5fc998b958dd629458 2024-11-08T19:13:26,729 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN in 617 msec 2024-11-08T19:13:26,729 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/9e50623b9f9d6b5fc998b958dd629458 2024-11-08T19:13:26,730 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/9e50623b9f9d6b5fc998b958dd629458 2024-11-08T19:13:26,730 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for 9e50623b9f9d6b5fc998b958dd629458 2024-11-08T19:13:26,730 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for e92192d7abb7a6c62f8815702d7dc38d 2024-11-08T19:13:26,730 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for 9e50623b9f9d6b5fc998b958dd629458 2024-11-08T19:13:26,730 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=caa4304cf54e018c1a1a2071373865b5, regionState=OPEN, openSeqNum=2, regionLocation=692b295ad45c,36007,1731093202744 2024-11-08T19:13:26,734 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=53b7061bfce2939f050c57221eed7f2a, regionState=OPEN, openSeqNum=2, regionLocation=692b295ad45c,39647,1731093203061 2024-11-08T19:13:26,734 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for 9e50623b9f9d6b5fc998b958dd629458 2024-11-08T19:13:26,734 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure caa4304cf54e018c1a1a2071373865b5, server=692b295ad45c,36007,1731093202744 because future has completed 2024-11-08T19:13:26,738 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 53b7061bfce2939f050c57221eed7f2a, server=692b295ad45c,39647,1731093203061 because future has completed 2024-11-08T19:13:26,738 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/9e50623b9f9d6b5fc998b958dd629458/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T19:13:26,738 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/e92192d7abb7a6c62f8815702d7dc38d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T19:13:26,738 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened e92192d7abb7a6c62f8815702d7dc38d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67988354, jitterRate=0.013105422258377075}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T19:13:26,738 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened 9e50623b9f9d6b5fc998b958dd629458; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66485655, jitterRate=-0.009286537766456604}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T19:13:26,739 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e92192d7abb7a6c62f8815702d7dc38d 2024-11-08T19:13:26,739 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9e50623b9f9d6b5fc998b958dd629458 2024-11-08T19:13:26,739 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for e92192d7abb7a6c62f8815702d7dc38d: Running coprocessor pre-open hook at 1731093206718Writing region info on filesystem at 1731093206718Initializing all the Stores at 1731093206720 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731093206720Cleaning up temporary data from old regions at 1731093206727 (+7 ms)Running coprocessor post-open hooks at 1731093206739 (+12 ms)Region opened successfully at 1731093206739 2024-11-08T19:13:26,739 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for 9e50623b9f9d6b5fc998b958dd629458: Running coprocessor pre-open hook at 1731093206723Writing region info on filesystem at 1731093206723Initializing all the Stores at 1731093206724 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731093206724Cleaning up temporary data from old regions at 1731093206730 (+6 ms)Running coprocessor post-open hooks at 1731093206739 (+9 ms)Region opened successfully at 1731093206739 2024-11-08T19:13:26,740 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1731093205731.e92192d7abb7a6c62f8815702d7dc38d., pid=37, masterSystemTime=1731093206439 2024-11-08T19:13:26,740 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1731093205731.9e50623b9f9d6b5fc998b958dd629458., pid=53, masterSystemTime=1731093206545 2024-11-08T19:13:26,740 INFO [StoreOpener-2548b6c044cd2c9ceb7dec340e2cde6c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2548b6c044cd2c9ceb7dec340e2cde6c 2024-11-08T19:13:26,743 INFO [StoreOpener-2548b6c044cd2c9ceb7dec340e2cde6c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2548b6c044cd2c9ceb7dec340e2cde6c columnFamilyName family 2024-11-08T19:13:26,743 DEBUG [StoreOpener-2548b6c044cd2c9ceb7dec340e2cde6c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:26,744 INFO [StoreOpener-2548b6c044cd2c9ceb7dec340e2cde6c-1 {}] regionserver.HStore(327): Store=2548b6c044cd2c9ceb7dec340e2cde6c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T19:13:26,744 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1731093205731.9e50623b9f9d6b5fc998b958dd629458. 2024-11-08T19:13:26,744 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,www,1731093205731.9e50623b9f9d6b5fc998b958dd629458. 2024-11-08T19:13:26,744 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 2548b6c044cd2c9ceb7dec340e2cde6c 2024-11-08T19:13:26,744 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,eee,1731093205731.fdc84e1c2d9bdd5f39d647a57ff91a05. 2024-11-08T19:13:26,744 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => fdc84e1c2d9bdd5f39d647a57ff91a05, NAME => 'test,eee,1731093205731.fdc84e1c2d9bdd5f39d647a57ff91a05.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-08T19:13:26,745 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test fdc84e1c2d9bdd5f39d647a57ff91a05 2024-11-08T19:13:26,745 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1731093205731.e92192d7abb7a6c62f8815702d7dc38d. 2024-11-08T19:13:26,745 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,ppp,1731093205731.e92192d7abb7a6c62f8815702d7dc38d. 2024-11-08T19:13:26,745 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,eee,1731093205731.fdc84e1c2d9bdd5f39d647a57ff91a05.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:26,745 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,jjj,1731093205731.f7a8c4f9e9448f30c950ddcf996ae56f. 2024-11-08T19:13:26,745 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for fdc84e1c2d9bdd5f39d647a57ff91a05 2024-11-08T19:13:26,745 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for fdc84e1c2d9bdd5f39d647a57ff91a05 2024-11-08T19:13:26,745 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => f7a8c4f9e9448f30c950ddcf996ae56f, NAME => 'test,jjj,1731093205731.f7a8c4f9e9448f30c950ddcf996ae56f.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-08T19:13:26,745 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/2548b6c044cd2c9ceb7dec340e2cde6c 2024-11-08T19:13:26,745 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f7a8c4f9e9448f30c950ddcf996ae56f 2024-11-08T19:13:26,745 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,jjj,1731093205731.f7a8c4f9e9448f30c950ddcf996ae56f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:26,745 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for f7a8c4f9e9448f30c950ddcf996ae56f 2024-11-08T19:13:26,745 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for f7a8c4f9e9448f30c950ddcf996ae56f 2024-11-08T19:13:26,746 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/2548b6c044cd2c9ceb7dec340e2cde6c 2024-11-08T19:13:26,746 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 2548b6c044cd2c9ceb7dec340e2cde6c 2024-11-08T19:13:26,746 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 2548b6c044cd2c9ceb7dec340e2cde6c 2024-11-08T19:13:26,748 INFO [StoreOpener-fdc84e1c2d9bdd5f39d647a57ff91a05-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region fdc84e1c2d9bdd5f39d647a57ff91a05 2024-11-08T19:13:26,750 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 2548b6c044cd2c9ceb7dec340e2cde6c 2024-11-08T19:13:26,750 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=10 2024-11-08T19:13:26,750 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 53b7061bfce2939f050c57221eed7f2a, server=692b295ad45c,39647,1731093203061 in 376 msec 2024-11-08T19:13:26,750 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39509 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=692b295ad45c,36007,1731093202744, table=test, region=07363d44bad100b56e1b1aba358b0ed2. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-08T19:13:26,750 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39509 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=692b295ad45c,36007,1731093202744, table=test, region=caa4304cf54e018c1a1a2071373865b5. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-08T19:13:26,752 INFO [StoreOpener-f7a8c4f9e9448f30c950ddcf996ae56f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f7a8c4f9e9448f30c950ddcf996ae56f 2024-11-08T19:13:26,752 INFO [StoreOpener-fdc84e1c2d9bdd5f39d647a57ff91a05-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fdc84e1c2d9bdd5f39d647a57ff91a05 columnFamilyName family 2024-11-08T19:13:26,752 DEBUG [StoreOpener-fdc84e1c2d9bdd5f39d647a57ff91a05-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:26,753 INFO [StoreOpener-fdc84e1c2d9bdd5f39d647a57ff91a05-1 {}] regionserver.HStore(327): Store=fdc84e1c2d9bdd5f39d647a57ff91a05/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T19:13:26,754 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for fdc84e1c2d9bdd5f39d647a57ff91a05 2024-11-08T19:13:26,756 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/fdc84e1c2d9bdd5f39d647a57ff91a05 2024-11-08T19:13:26,756 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN in 643 msec 2024-11-08T19:13:26,756 INFO [StoreOpener-f7a8c4f9e9448f30c950ddcf996ae56f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7a8c4f9e9448f30c950ddcf996ae56f columnFamilyName family 2024-11-08T19:13:26,756 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/fdc84e1c2d9bdd5f39d647a57ff91a05 2024-11-08T19:13:26,756 DEBUG [StoreOpener-f7a8c4f9e9448f30c950ddcf996ae56f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:26,756 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/2548b6c044cd2c9ceb7dec340e2cde6c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T19:13:26,757 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for fdc84e1c2d9bdd5f39d647a57ff91a05 2024-11-08T19:13:26,757 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for fdc84e1c2d9bdd5f39d647a57ff91a05 2024-11-08T19:13:26,757 INFO [StoreOpener-f7a8c4f9e9448f30c950ddcf996ae56f-1 {}] regionserver.HStore(327): Store=f7a8c4f9e9448f30c950ddcf996ae56f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T19:13:26,757 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 2548b6c044cd2c9ceb7dec340e2cde6c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62462298, jitterRate=-0.06923922896385193}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T19:13:26,757 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2548b6c044cd2c9ceb7dec340e2cde6c 2024-11-08T19:13:26,757 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for f7a8c4f9e9448f30c950ddcf996ae56f 2024-11-08T19:13:26,757 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 2548b6c044cd2c9ceb7dec340e2cde6c: Running coprocessor pre-open hook at 1731093206723Writing region info on filesystem at 1731093206723Initializing all the Stores at 1731093206726 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731093206726Cleaning up temporary data from old regions at 1731093206747 (+21 ms)Running coprocessor post-open hooks at 1731093206757 (+10 ms)Region opened successfully at 1731093206757 2024-11-08T19:13:26,759 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/f7a8c4f9e9448f30c950ddcf996ae56f 2024-11-08T19:13:26,759 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=e92192d7abb7a6c62f8815702d7dc38d, regionState=OPEN, openSeqNum=2, regionLocation=692b295ad45c,42693,1731093202977 2024-11-08T19:13:26,759 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1731093205731.2548b6c044cd2c9ceb7dec340e2cde6c., pid=44, masterSystemTime=1731093206512 2024-11-08T19:13:26,759 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/f7a8c4f9e9448f30c950ddcf996ae56f 2024-11-08T19:13:26,760 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for fdc84e1c2d9bdd5f39d647a57ff91a05 2024-11-08T19:13:26,760 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for f7a8c4f9e9448f30c950ddcf996ae56f 2024-11-08T19:13:26,760 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for f7a8c4f9e9448f30c950ddcf996ae56f 2024-11-08T19:13:26,763 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1731093205731.2548b6c044cd2c9ceb7dec340e2cde6c. 2024-11-08T19:13:26,763 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,iii,1731093205731.2548b6c044cd2c9ceb7dec340e2cde6c. 2024-11-08T19:13:26,763 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,lll,1731093205731.6e461a86c03ca80ab184602fce3cc56c. 2024-11-08T19:13:26,763 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 6e461a86c03ca80ab184602fce3cc56c, NAME => 'test,lll,1731093205731.6e461a86c03ca80ab184602fce3cc56c.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-08T19:13:26,764 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6e461a86c03ca80ab184602fce3cc56c 2024-11-08T19:13:26,764 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,lll,1731093205731.6e461a86c03ca80ab184602fce3cc56c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:26,764 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 6e461a86c03ca80ab184602fce3cc56c 2024-11-08T19:13:26,764 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 6e461a86c03ca80ab184602fce3cc56c 2024-11-08T19:13:26,764 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure e92192d7abb7a6c62f8815702d7dc38d, server=692b295ad45c,42693,1731093202977 because future has completed 2024-11-08T19:13:26,764 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for f7a8c4f9e9448f30c950ddcf996ae56f 2024-11-08T19:13:26,765 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=2548b6c044cd2c9ceb7dec340e2cde6c, regionState=OPEN, openSeqNum=2, regionLocation=692b295ad45c,39647,1731093203061 2024-11-08T19:13:26,765 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/fdc84e1c2d9bdd5f39d647a57ff91a05/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T19:13:26,766 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened fdc84e1c2d9bdd5f39d647a57ff91a05; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59963647, jitterRate=-0.10647203028202057}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T19:13:26,766 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fdc84e1c2d9bdd5f39d647a57ff91a05 2024-11-08T19:13:26,766 INFO [StoreOpener-6e461a86c03ca80ab184602fce3cc56c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6e461a86c03ca80ab184602fce3cc56c 2024-11-08T19:13:26,766 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for fdc84e1c2d9bdd5f39d647a57ff91a05: Running coprocessor pre-open hook at 1731093206745Writing region info on filesystem at 1731093206745Initializing all the Stores at 1731093206746 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731093206746Cleaning up temporary data from old regions at 1731093206757 (+11 ms)Running coprocessor post-open hooks at 1731093206766 (+9 ms)Region opened successfully at 1731093206766 2024-11-08T19:13:26,768 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1731093205731.fdc84e1c2d9bdd5f39d647a57ff91a05., pid=56, masterSystemTime=1731093206545 2024-11-08T19:13:26,769 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2548b6c044cd2c9ceb7dec340e2cde6c, server=692b295ad45c,39647,1731093203061 because future has completed 2024-11-08T19:13:26,770 INFO [StoreOpener-6e461a86c03ca80ab184602fce3cc56c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6e461a86c03ca80ab184602fce3cc56c columnFamilyName family 2024-11-08T19:13:26,770 DEBUG [StoreOpener-6e461a86c03ca80ab184602fce3cc56c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:26,771 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1731093205731.fdc84e1c2d9bdd5f39d647a57ff91a05. 2024-11-08T19:13:26,771 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,eee,1731093205731.fdc84e1c2d9bdd5f39d647a57ff91a05. 2024-11-08T19:13:26,771 INFO [StoreOpener-6e461a86c03ca80ab184602fce3cc56c-1 {}] regionserver.HStore(327): Store=6e461a86c03ca80ab184602fce3cc56c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T19:13:26,771 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,zzz,1731093205731.2afe1e3e862a406c7fcb359dc1b60d57. 2024-11-08T19:13:26,772 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 6e461a86c03ca80ab184602fce3cc56c 2024-11-08T19:13:26,772 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => 2afe1e3e862a406c7fcb359dc1b60d57, NAME => 'test,zzz,1731093205731.2afe1e3e862a406c7fcb359dc1b60d57.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-08T19:13:26,772 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2afe1e3e862a406c7fcb359dc1b60d57 2024-11-08T19:13:26,772 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,zzz,1731093205731.2afe1e3e862a406c7fcb359dc1b60d57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:26,772 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for 2afe1e3e862a406c7fcb359dc1b60d57 2024-11-08T19:13:26,772 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for 2afe1e3e862a406c7fcb359dc1b60d57 2024-11-08T19:13:26,773 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/6e461a86c03ca80ab184602fce3cc56c 2024-11-08T19:13:26,773 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/6e461a86c03ca80ab184602fce3cc56c 2024-11-08T19:13:26,774 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 6e461a86c03ca80ab184602fce3cc56c 2024-11-08T19:13:26,774 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=20 2024-11-08T19:13:26,774 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 6e461a86c03ca80ab184602fce3cc56c 2024-11-08T19:13:26,774 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure e92192d7abb7a6c62f8815702d7dc38d, server=692b295ad45c,42693,1731093202977 in 443 msec 2024-11-08T19:13:26,774 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/f7a8c4f9e9448f30c950ddcf996ae56f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T19:13:26,775 INFO [StoreOpener-2afe1e3e862a406c7fcb359dc1b60d57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2afe1e3e862a406c7fcb359dc1b60d57 2024-11-08T19:13:26,775 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened f7a8c4f9e9448f30c950ddcf996ae56f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59361946, jitterRate=-0.1154380738735199}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T19:13:26,775 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f7a8c4f9e9448f30c950ddcf996ae56f 2024-11-08T19:13:26,775 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for f7a8c4f9e9448f30c950ddcf996ae56f: Running coprocessor pre-open hook at 1731093206746Writing region info on filesystem at 1731093206746Initializing all the Stores at 1731093206748 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731093206748Cleaning up temporary data from old regions at 1731093206760 (+12 ms)Running coprocessor post-open hooks at 1731093206775 (+15 ms)Region opened successfully at 1731093206775 2024-11-08T19:13:26,777 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1731093205731.f7a8c4f9e9448f30c950ddcf996ae56f., pid=36, masterSystemTime=1731093206439 2024-11-08T19:13:26,777 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 6e461a86c03ca80ab184602fce3cc56c 2024-11-08T19:13:26,778 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e92192d7abb7a6c62f8815702d7dc38d, ASSIGN in 667 msec 2024-11-08T19:13:26,780 INFO [StoreOpener-2afe1e3e862a406c7fcb359dc1b60d57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2afe1e3e862a406c7fcb359dc1b60d57 columnFamilyName family 2024-11-08T19:13:26,780 DEBUG [StoreOpener-2afe1e3e862a406c7fcb359dc1b60d57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:26,781 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=fdc84e1c2d9bdd5f39d647a57ff91a05, regionState=OPEN, openSeqNum=2, regionLocation=692b295ad45c,36007,1731093202744 2024-11-08T19:13:26,784 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure fdc84e1c2d9bdd5f39d647a57ff91a05, server=692b295ad45c,36007,1731093202744 because future has completed 2024-11-08T19:13:26,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=13 2024-11-08T19:13:26,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure 2548b6c044cd2c9ceb7dec340e2cde6c, server=692b295ad45c,39647,1731093203061 in 416 msec 2024-11-08T19:13:26,789 INFO [StoreOpener-2afe1e3e862a406c7fcb359dc1b60d57-1 {}] regionserver.HStore(327): Store=2afe1e3e862a406c7fcb359dc1b60d57/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T19:13:26,789 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for 2afe1e3e862a406c7fcb359dc1b60d57 2024-11-08T19:13:26,790 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1731093205731.f7a8c4f9e9448f30c950ddcf996ae56f. 2024-11-08T19:13:26,790 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,jjj,1731093205731.f7a8c4f9e9448f30c950ddcf996ae56f. 2024-11-08T19:13:26,791 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/2afe1e3e862a406c7fcb359dc1b60d57 2024-11-08T19:13:26,791 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/2afe1e3e862a406c7fcb359dc1b60d57 2024-11-08T19:13:26,792 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for 2afe1e3e862a406c7fcb359dc1b60d57 2024-11-08T19:13:26,792 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for 2afe1e3e862a406c7fcb359dc1b60d57 2024-11-08T19:13:26,793 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/6e461a86c03ca80ab184602fce3cc56c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T19:13:26,795 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 6e461a86c03ca80ab184602fce3cc56c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70115810, jitterRate=0.044806987047195435}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T19:13:26,795 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6e461a86c03ca80ab184602fce3cc56c 2024-11-08T19:13:26,795 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 6e461a86c03ca80ab184602fce3cc56c: Running coprocessor pre-open hook at 1731093206764Writing region info on filesystem at 1731093206764Initializing all the Stores at 1731093206765 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731093206765Cleaning up temporary data from old regions at 1731093206774 (+9 ms)Running coprocessor post-open hooks at 1731093206795 (+21 ms)Region opened successfully at 1731093206795 2024-11-08T19:13:26,795 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for 2afe1e3e862a406c7fcb359dc1b60d57 2024-11-08T19:13:26,796 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN in 682 msec 2024-11-08T19:13:26,796 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1731093205731.6e461a86c03ca80ab184602fce3cc56c., pid=41, masterSystemTime=1731093206512 2024-11-08T19:13:26,800 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=f7a8c4f9e9448f30c950ddcf996ae56f, regionState=OPEN, openSeqNum=2, regionLocation=692b295ad45c,42693,1731093202977 2024-11-08T19:13:26,800 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1731093205731.6e461a86c03ca80ab184602fce3cc56c. 2024-11-08T19:13:26,800 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,lll,1731093205731.6e461a86c03ca80ab184602fce3cc56c. 2024-11-08T19:13:26,800 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,ooo,1731093205731.0095b58ae56790b54bf50c65ac0754fa. 2024-11-08T19:13:26,801 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => 0095b58ae56790b54bf50c65ac0754fa, NAME => 'test,ooo,1731093205731.0095b58ae56790b54bf50c65ac0754fa.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-08T19:13:26,801 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0095b58ae56790b54bf50c65ac0754fa 2024-11-08T19:13:26,801 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,ooo,1731093205731.0095b58ae56790b54bf50c65ac0754fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:26,801 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for 0095b58ae56790b54bf50c65ac0754fa 2024-11-08T19:13:26,801 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for 0095b58ae56790b54bf50c65ac0754fa 2024-11-08T19:13:26,804 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure f7a8c4f9e9448f30c950ddcf996ae56f, server=692b295ad45c,42693,1731093202977 because future has completed 2024-11-08T19:13:26,805 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=6e461a86c03ca80ab184602fce3cc56c, regionState=OPEN, openSeqNum=2, regionLocation=692b295ad45c,39647,1731093203061 2024-11-08T19:13:26,808 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6e461a86c03ca80ab184602fce3cc56c, server=692b295ad45c,39647,1731093203061 because future has completed 2024-11-08T19:13:26,813 INFO [StoreOpener-0095b58ae56790b54bf50c65ac0754fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0095b58ae56790b54bf50c65ac0754fa 2024-11-08T19:13:26,813 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/2afe1e3e862a406c7fcb359dc1b60d57/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T19:13:26,814 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=14 2024-11-08T19:13:26,814 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure f7a8c4f9e9448f30c950ddcf996ae56f, server=692b295ad45c,42693,1731093202977 in 488 msec 2024-11-08T19:13:26,814 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened 2afe1e3e862a406c7fcb359dc1b60d57; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65955260, jitterRate=-0.01719003915786743}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T19:13:26,815 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2afe1e3e862a406c7fcb359dc1b60d57 2024-11-08T19:13:26,815 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for 2afe1e3e862a406c7fcb359dc1b60d57: Running coprocessor pre-open hook at 1731093206773Writing region info on filesystem at 1731093206773Initializing all the Stores at 1731093206774 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731093206774Cleaning up temporary data from old regions at 1731093206792 (+18 ms)Running coprocessor post-open hooks at 1731093206815 (+23 ms)Region opened successfully at 1731093206815 2024-11-08T19:13:26,816 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1731093205731.2afe1e3e862a406c7fcb359dc1b60d57., pid=48, masterSystemTime=1731093206545 2024-11-08T19:13:26,818 INFO [StoreOpener-0095b58ae56790b54bf50c65ac0754fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0095b58ae56790b54bf50c65ac0754fa columnFamilyName family 2024-11-08T19:13:26,818 DEBUG [StoreOpener-0095b58ae56790b54bf50c65ac0754fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:26,818 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f7a8c4f9e9448f30c950ddcf996ae56f, ASSIGN in 707 msec 2024-11-08T19:13:26,819 INFO [StoreOpener-0095b58ae56790b54bf50c65ac0754fa-1 {}] regionserver.HStore(327): Store=0095b58ae56790b54bf50c65ac0754fa/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T19:13:26,820 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for 0095b58ae56790b54bf50c65ac0754fa 2024-11-08T19:13:26,820 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1731093205731.2afe1e3e862a406c7fcb359dc1b60d57. 2024-11-08T19:13:26,820 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,zzz,1731093205731.2afe1e3e862a406c7fcb359dc1b60d57. 2024-11-08T19:13:26,820 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,ttt,1731093205731.1918a8713e3d1c571d1188f35d75a101. 2024-11-08T19:13:26,821 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => 1918a8713e3d1c571d1188f35d75a101, NAME => 'test,ttt,1731093205731.1918a8713e3d1c571d1188f35d75a101.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-08T19:13:26,821 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1918a8713e3d1c571d1188f35d75a101 2024-11-08T19:13:26,821 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,ttt,1731093205731.1918a8713e3d1c571d1188f35d75a101.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:26,821 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for 1918a8713e3d1c571d1188f35d75a101 2024-11-08T19:13:26,821 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for 1918a8713e3d1c571d1188f35d75a101 2024-11-08T19:13:26,824 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/0095b58ae56790b54bf50c65ac0754fa 2024-11-08T19:13:26,824 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/0095b58ae56790b54bf50c65ac0754fa 2024-11-08T19:13:26,825 INFO [StoreOpener-1918a8713e3d1c571d1188f35d75a101-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1918a8713e3d1c571d1188f35d75a101 2024-11-08T19:13:26,825 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for 0095b58ae56790b54bf50c65ac0754fa 2024-11-08T19:13:26,825 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for 0095b58ae56790b54bf50c65ac0754fa 2024-11-08T19:13:26,828 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=16 2024-11-08T19:13:26,828 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 6e461a86c03ca80ab184602fce3cc56c, server=692b295ad45c,39647,1731093203061 in 464 msec 2024-11-08T19:13:26,829 INFO [StoreOpener-1918a8713e3d1c571d1188f35d75a101-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1918a8713e3d1c571d1188f35d75a101 columnFamilyName family 2024-11-08T19:13:26,829 DEBUG [StoreOpener-1918a8713e3d1c571d1188f35d75a101-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:26,829 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for 0095b58ae56790b54bf50c65ac0754fa 2024-11-08T19:13:26,830 INFO [StoreOpener-1918a8713e3d1c571d1188f35d75a101-1 {}] regionserver.HStore(327): Store=1918a8713e3d1c571d1188f35d75a101/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T19:13:26,830 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for 1918a8713e3d1c571d1188f35d75a101 2024-11-08T19:13:26,832 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/1918a8713e3d1c571d1188f35d75a101 2024-11-08T19:13:26,833 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/1918a8713e3d1c571d1188f35d75a101 2024-11-08T19:13:26,833 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/0095b58ae56790b54bf50c65ac0754fa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T19:13:26,833 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN in 721 msec 2024-11-08T19:13:26,834 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for 1918a8713e3d1c571d1188f35d75a101 2024-11-08T19:13:26,834 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for 1918a8713e3d1c571d1188f35d75a101 2024-11-08T19:13:26,834 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened 0095b58ae56790b54bf50c65ac0754fa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70132882, jitterRate=0.0450613796710968}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T19:13:26,834 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0095b58ae56790b54bf50c65ac0754fa 2024-11-08T19:13:26,834 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for 0095b58ae56790b54bf50c65ac0754fa: Running coprocessor pre-open hook at 1731093206802Writing region info on filesystem at 1731093206802Initializing all the Stores at 1731093206804 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731093206804Cleaning up temporary data from old regions at 1731093206825 (+21 ms)Running coprocessor post-open hooks at 1731093206834 (+9 ms)Region opened successfully at 1731093206834 2024-11-08T19:13:26,836 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=2afe1e3e862a406c7fcb359dc1b60d57, regionState=OPEN, openSeqNum=2, regionLocation=692b295ad45c,36007,1731093202744 2024-11-08T19:13:26,836 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1731093205731.0095b58ae56790b54bf50c65ac0754fa., pid=45, masterSystemTime=1731093206512 2024-11-08T19:13:26,839 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for 1918a8713e3d1c571d1188f35d75a101 2024-11-08T19:13:26,839 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2afe1e3e862a406c7fcb359dc1b60d57, server=692b295ad45c,36007,1731093202744 because future has completed 2024-11-08T19:13:26,840 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1731093205731.0095b58ae56790b54bf50c65ac0754fa. 2024-11-08T19:13:26,840 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,ooo,1731093205731.0095b58ae56790b54bf50c65ac0754fa. 2024-11-08T19:13:26,843 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/1918a8713e3d1c571d1188f35d75a101/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T19:13:26,843 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened 1918a8713e3d1c571d1188f35d75a101; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60810023, jitterRate=-0.09386004507541656}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T19:13:26,844 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1918a8713e3d1c571d1188f35d75a101 2024-11-08T19:13:26,844 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for 1918a8713e3d1c571d1188f35d75a101: Running coprocessor pre-open hook at 1731093206822Writing region info on filesystem at 1731093206822Initializing all the Stores at 1731093206823 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731093206823Cleaning up temporary data from old regions at 1731093206834 (+11 ms)Running coprocessor post-open hooks at 1731093206844 (+10 ms)Region opened successfully at 1731093206844 2024-11-08T19:13:26,844 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=9 2024-11-08T19:13:26,844 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure fdc84e1c2d9bdd5f39d647a57ff91a05, server=692b295ad45c,36007,1731093202744 in 409 msec 2024-11-08T19:13:26,845 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1731093205731.1918a8713e3d1c571d1188f35d75a101., pid=52, masterSystemTime=1731093206545 2024-11-08T19:13:26,847 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fdc84e1c2d9bdd5f39d647a57ff91a05, ASSIGN in 737 msec 2024-11-08T19:13:26,847 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1731093205731.1918a8713e3d1c571d1188f35d75a101. 2024-11-08T19:13:26,847 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,ttt,1731093205731.1918a8713e3d1c571d1188f35d75a101. 2024-11-08T19:13:26,847 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,hhh,1731093205731.7d1d9fe849fd45b830c330834b58d508. 2024-11-08T19:13:26,848 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => 7d1d9fe849fd45b830c330834b58d508, NAME => 'test,hhh,1731093205731.7d1d9fe849fd45b830c330834b58d508.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-08T19:13:26,848 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=1918a8713e3d1c571d1188f35d75a101, regionState=OPEN, openSeqNum=2, regionLocation=692b295ad45c,36007,1731093202744 2024-11-08T19:13:26,848 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7d1d9fe849fd45b830c330834b58d508 2024-11-08T19:13:26,848 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,hhh,1731093205731.7d1d9fe849fd45b830c330834b58d508.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:26,848 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for 7d1d9fe849fd45b830c330834b58d508 2024-11-08T19:13:26,848 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for 7d1d9fe849fd45b830c330834b58d508 2024-11-08T19:13:26,851 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1918a8713e3d1c571d1188f35d75a101, server=692b295ad45c,36007,1731093202744 because future has completed 2024-11-08T19:13:26,851 INFO [StoreOpener-7d1d9fe849fd45b830c330834b58d508-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7d1d9fe849fd45b830c330834b58d508 2024-11-08T19:13:26,854 INFO [StoreOpener-7d1d9fe849fd45b830c330834b58d508-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7d1d9fe849fd45b830c330834b58d508 columnFamilyName family 2024-11-08T19:13:26,854 DEBUG [StoreOpener-7d1d9fe849fd45b830c330834b58d508-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:26,855 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39509 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=692b295ad45c,36007,1731093202744, table=test, region=07363d44bad100b56e1b1aba358b0ed2. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-08T19:13:26,855 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39509 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=692b295ad45c,36007,1731093202744, table=test, region=caa4304cf54e018c1a1a2071373865b5. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-08T19:13:26,855 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39509 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=692b295ad45c,36007,1731093202744, table=test, region=1918a8713e3d1c571d1188f35d75a101. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-08T19:13:26,856 INFO [StoreOpener-7d1d9fe849fd45b830c330834b58d508-1 {}] regionserver.HStore(327): Store=7d1d9fe849fd45b830c330834b58d508/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T19:13:26,856 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for 7d1d9fe849fd45b830c330834b58d508 2024-11-08T19:13:26,858 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/7d1d9fe849fd45b830c330834b58d508 2024-11-08T19:13:26,858 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/7d1d9fe849fd45b830c330834b58d508 2024-11-08T19:13:26,858 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=30 2024-11-08T19:13:26,859 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure 2afe1e3e862a406c7fcb359dc1b60d57, server=692b295ad45c,36007,1731093202744 in 462 msec 2024-11-08T19:13:26,859 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for 7d1d9fe849fd45b830c330834b58d508 2024-11-08T19:13:26,859 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for 7d1d9fe849fd45b830c330834b58d508 2024-11-08T19:13:26,862 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=2afe1e3e862a406c7fcb359dc1b60d57, ASSIGN in 752 msec 2024-11-08T19:13:26,862 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for 7d1d9fe849fd45b830c330834b58d508 2024-11-08T19:13:26,867 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/7d1d9fe849fd45b830c330834b58d508/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T19:13:26,868 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened 7d1d9fe849fd45b830c330834b58d508; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71241304, jitterRate=0.06157815456390381}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T19:13:26,868 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7d1d9fe849fd45b830c330834b58d508 2024-11-08T19:13:26,868 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for 7d1d9fe849fd45b830c330834b58d508: Running coprocessor pre-open hook at 1731093206849Writing region info on filesystem at 1731093206849Initializing all the Stores at 1731093206850 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731093206850Cleaning up temporary data from old regions at 1731093206859 (+9 ms)Running coprocessor post-open hooks at 1731093206868 (+9 ms)Region opened successfully at 1731093206868 2024-11-08T19:13:26,868 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=24 2024-11-08T19:13:26,868 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 1918a8713e3d1c571d1188f35d75a101, server=692b295ad45c,36007,1731093202744 in 452 msec 2024-11-08T19:13:26,869 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1731093205731.7d1d9fe849fd45b830c330834b58d508., pid=54, masterSystemTime=1731093206545 2024-11-08T19:13:26,871 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=1918a8713e3d1c571d1188f35d75a101, ASSIGN in 762 msec 2024-11-08T19:13:26,872 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1731093205731.7d1d9fe849fd45b830c330834b58d508. 2024-11-08T19:13:26,872 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,hhh,1731093205731.7d1d9fe849fd45b830c330834b58d508. 2024-11-08T19:13:26,872 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,kkk,1731093205731.3d0557ff953666f2f97a7be0e2d0bc87. 2024-11-08T19:13:26,872 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => 3d0557ff953666f2f97a7be0e2d0bc87, NAME => 'test,kkk,1731093205731.3d0557ff953666f2f97a7be0e2d0bc87.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-08T19:13:26,872 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=7d1d9fe849fd45b830c330834b58d508, regionState=OPEN, openSeqNum=2, regionLocation=692b295ad45c,36007,1731093202744 2024-11-08T19:13:26,872 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 3d0557ff953666f2f97a7be0e2d0bc87 2024-11-08T19:13:26,872 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,kkk,1731093205731.3d0557ff953666f2f97a7be0e2d0bc87.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:26,873 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for 3d0557ff953666f2f97a7be0e2d0bc87 2024-11-08T19:13:26,873 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for 3d0557ff953666f2f97a7be0e2d0bc87 2024-11-08T19:13:26,875 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7d1d9fe849fd45b830c330834b58d508, server=692b295ad45c,36007,1731093202744 because future has completed 2024-11-08T19:13:26,875 INFO [StoreOpener-3d0557ff953666f2f97a7be0e2d0bc87-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 3d0557ff953666f2f97a7be0e2d0bc87 2024-11-08T19:13:26,876 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=0095b58ae56790b54bf50c65ac0754fa, regionState=OPEN, openSeqNum=2, regionLocation=692b295ad45c,39647,1731093203061 2024-11-08T19:13:26,878 INFO [StoreOpener-3d0557ff953666f2f97a7be0e2d0bc87-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3d0557ff953666f2f97a7be0e2d0bc87 columnFamilyName family 2024-11-08T19:13:26,878 DEBUG [StoreOpener-3d0557ff953666f2f97a7be0e2d0bc87-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:26,879 INFO [StoreOpener-3d0557ff953666f2f97a7be0e2d0bc87-1 {}] regionserver.HStore(327): Store=3d0557ff953666f2f97a7be0e2d0bc87/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T19:13:26,879 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for 3d0557ff953666f2f97a7be0e2d0bc87 2024-11-08T19:13:26,879 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0095b58ae56790b54bf50c65ac0754fa, server=692b295ad45c,39647,1731093203061 because future has completed 2024-11-08T19:13:26,880 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/3d0557ff953666f2f97a7be0e2d0bc87 2024-11-08T19:13:26,881 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/3d0557ff953666f2f97a7be0e2d0bc87 2024-11-08T19:13:26,881 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for 3d0557ff953666f2f97a7be0e2d0bc87 2024-11-08T19:13:26,881 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for 3d0557ff953666f2f97a7be0e2d0bc87 2024-11-08T19:13:26,884 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for 3d0557ff953666f2f97a7be0e2d0bc87 2024-11-08T19:13:26,884 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=12 2024-11-08T19:13:26,884 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 7d1d9fe849fd45b830c330834b58d508, server=692b295ad45c,36007,1731093202744 in 457 msec 2024-11-08T19:13:26,887 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/3d0557ff953666f2f97a7be0e2d0bc87/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T19:13:26,887 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7d1d9fe849fd45b830c330834b58d508, ASSIGN in 778 msec 2024-11-08T19:13:26,888 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened 3d0557ff953666f2f97a7be0e2d0bc87; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59430410, jitterRate=-0.11441788077354431}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T19:13:26,888 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3d0557ff953666f2f97a7be0e2d0bc87 2024-11-08T19:13:26,889 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for 3d0557ff953666f2f97a7be0e2d0bc87: Running coprocessor pre-open hook at 1731093206873Writing region info on filesystem at 1731093206873Initializing all the Stores at 1731093206875 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731093206875Cleaning up temporary data from old regions at 1731093206881 (+6 ms)Running coprocessor post-open hooks at 1731093206888 (+7 ms)Region opened successfully at 1731093206888 2024-11-08T19:13:26,889 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1731093205731.3d0557ff953666f2f97a7be0e2d0bc87., pid=50, masterSystemTime=1731093206545 2024-11-08T19:13:26,892 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=19 2024-11-08T19:13:26,892 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 0095b58ae56790b54bf50c65ac0754fa, server=692b295ad45c,39647,1731093203061 in 516 msec 2024-11-08T19:13:26,892 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1731093205731.3d0557ff953666f2f97a7be0e2d0bc87. 2024-11-08T19:13:26,892 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,kkk,1731093205731.3d0557ff953666f2f97a7be0e2d0bc87. 2024-11-08T19:13:26,893 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=3d0557ff953666f2f97a7be0e2d0bc87, regionState=OPEN, openSeqNum=2, regionLocation=692b295ad45c,36007,1731093202744 2024-11-08T19:13:26,895 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3d0557ff953666f2f97a7be0e2d0bc87, server=692b295ad45c,36007,1731093202744 because future has completed 2024-11-08T19:13:26,897 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN in 788 msec 2024-11-08T19:13:26,900 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=15 2024-11-08T19:13:26,900 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure 3d0557ff953666f2f97a7be0e2d0bc87, server=692b295ad45c,36007,1731093202744 in 494 msec 2024-11-08T19:13:26,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T19:13:26,902 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=3d0557ff953666f2f97a7be0e2d0bc87, ASSIGN in 793 msec 2024-11-08T19:13:26,904 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=9e50623b9f9d6b5fc998b958dd629458, regionState=OPEN, openSeqNum=2, regionLocation=692b295ad45c,36007,1731093202744 2024-11-08T19:13:26,906 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9e50623b9f9d6b5fc998b958dd629458, server=692b295ad45c,36007,1731093202744 because future has completed 2024-11-08T19:13:26,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=21 2024-11-08T19:13:26,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure caa4304cf54e018c1a1a2071373865b5, server=692b295ad45c,36007,1731093202744 in 508 msec 2024-11-08T19:13:26,912 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=caa4304cf54e018c1a1a2071373865b5, ASSIGN in 803 msec 2024-11-08T19:13:26,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=27 2024-11-08T19:13:26,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 9e50623b9f9d6b5fc998b958dd629458, server=692b295ad45c,36007,1731093202744 in 498 msec 2024-11-08T19:13:26,919 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=9e50623b9f9d6b5fc998b958dd629458, ASSIGN in 810 msec 2024-11-08T19:13:26,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=18 2024-11-08T19:13:26,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure 07363d44bad100b56e1b1aba358b0ed2, server=692b295ad45c,36007,1731093202744 in 512 msec 2024-11-08T19:13:26,925 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=07363d44bad100b56e1b1aba358b0ed2, ASSIGN in 816 msec 2024-11-08T19:13:26,926 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=7b4a23d5efe8e832c0d5cb16df3b1369, regionState=OPEN, openSeqNum=2, regionLocation=692b295ad45c,42693,1731093202977 2024-11-08T19:13:26,929 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7b4a23d5efe8e832c0d5cb16df3b1369, server=692b295ad45c,42693,1731093202977 because future has completed 2024-11-08T19:13:26,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=7 2024-11-08T19:13:26,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 7b4a23d5efe8e832c0d5cb16df3b1369, server=692b295ad45c,42693,1731093202977 in 628 msec 2024-11-08T19:13:26,937 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=7, resume processing ppid=4 2024-11-08T19:13:26,937 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7b4a23d5efe8e832c0d5cb16df3b1369, ASSIGN in 826 msec 2024-11-08T19:13:26,938 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T19:13:26,939 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731093206939"}]},"ts":"1731093206939"} 2024-11-08T19:13:26,942 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-11-08T19:13:26,943 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T19:13:26,949 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 1.2050 sec 2024-11-08T19:13:27,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T19:13:27,912 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-11-08T19:13:27,912 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-11-08T19:13:27,914 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T19:13:27,937 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-11-08T19:13:27,937 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T19:13:27,938 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-11-08T19:13:27,938 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T19:13:27,944 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-08T19:13:27,967 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-08T19:13:27,982 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T19:13:27,985 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T19:13:27,990 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T19:13:27,993 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T19:13:28,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d2ea8472f395c484d4998844c4dcd304 changed from -1.0 to 0.0, refreshing cache 2024-11-08T19:13:28,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region fdc84e1c2d9bdd5f39d647a57ff91a05 changed from -1.0 to 0.0, refreshing cache 2024-11-08T19:13:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 7d1d9fe849fd45b830c330834b58d508 changed from -1.0 to 0.0, refreshing cache 2024-11-08T19:13:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 3d0557ff953666f2f97a7be0e2d0bc87 changed from -1.0 to 0.0, refreshing cache 2024-11-08T19:13:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 07363d44bad100b56e1b1aba358b0ed2 changed from -1.0 to 0.0, refreshing cache 2024-11-08T19:13:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region caa4304cf54e018c1a1a2071373865b5 changed from -1.0 to 0.0, refreshing cache 2024-11-08T19:13:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1918a8713e3d1c571d1188f35d75a101 changed from -1.0 to 0.0, refreshing cache 2024-11-08T19:13:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 9e50623b9f9d6b5fc998b958dd629458 changed from -1.0 to 0.0, refreshing cache 2024-11-08T19:13:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2afe1e3e862a406c7fcb359dc1b60d57 changed from -1.0 to 0.0, refreshing cache 2024-11-08T19:13:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8cb2ac5239e07e076f513604c29c2ca8 changed from -1.0 to 0.0, refreshing cache 2024-11-08T19:13:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 53b7061bfce2939f050c57221eed7f2a changed from -1.0 to 0.0, refreshing cache 2024-11-08T19:13:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2548b6c044cd2c9ceb7dec340e2cde6c changed from -1.0 to 0.0, refreshing cache 2024-11-08T19:13:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6e461a86c03ca80ab184602fce3cc56c changed from -1.0 to 0.0, refreshing cache 2024-11-08T19:13:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0095b58ae56790b54bf50c65ac0754fa changed from -1.0 to 0.0, refreshing cache 2024-11-08T19:13:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f1496a2d5d6037cbe8d2f7a8f1407184 changed from -1.0 to 0.0, refreshing cache 2024-11-08T19:13:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 234c6774a52fc6a092eb269a4a390dc7 changed from -1.0 to 0.0, refreshing cache 2024-11-08T19:13:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 22cccf3cc4c2d3343ceae07ff16f899f changed from -1.0 to 0.0, refreshing cache 2024-11-08T19:13:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 7b4a23d5efe8e832c0d5cb16df3b1369 changed from -1.0 to 0.0, refreshing cache 2024-11-08T19:13:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a8e891c0073a08913834761f455a77e8 changed from -1.0 to 0.0, refreshing cache 2024-11-08T19:13:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ab92690881683c86166f7f23976b5188 changed from -1.0 to 0.0, refreshing cache 2024-11-08T19:13:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f7a8c4f9e9448f30c950ddcf996ae56f changed from -1.0 to 0.0, refreshing cache 2024-11-08T19:13:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 245339f649708a4ea812249dd3f822c4 changed from -1.0 to 0.0, refreshing cache 2024-11-08T19:13:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e92192d7abb7a6c62f8815702d7dc38d changed from -1.0 to 0.0, refreshing cache 2024-11-08T19:13:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 77e3781e625f0b456041758476a7d6b2 changed from -1.0 to 0.0, refreshing cache 2024-11-08T19:13:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f7c9c940725de9fafd49c71f7bebc530 changed from -1.0 to 0.0, refreshing cache 2024-11-08T19:13:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 48bb25f64ce0dca8172688cf9a77f57b changed from -1.0 to 0.0, refreshing cache 2024-11-08T19:13:28,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-11-08T19:13:28,047 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-11-08T19:13:28,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-11-08T19:13:28,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] balancer.BalancerClusterState(204): Hosts are {692b295ad45c=0} racks are {/default-rack=0} 2024-11-08T19:13:28,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] balancer.BalancerClusterState(303): server 0 has 9 regions 2024-11-08T19:13:28,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] balancer.BalancerClusterState(303): server 1 has 9 regions 2024-11-08T19:13:28,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] balancer.BalancerClusterState(303): server 2 has 9 regions 2024-11-08T19:13:28,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T19:13:28,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T19:13:28,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-08T19:13:28,049 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T19:13:28,049 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T19:13:28,049 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-08T19:13:28,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-11-08T19:13:28,053 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] balancer.StochasticLoadBalancer(395): Cluster wide - skipping load balancing because weighted average imbalance=0.016474464579901153 <= threshold(0.025). If you want more aggressive balancing, either lower hbase.master.balancer.stochastic.minCostNeedBalance from 0.025 or increase the relative multiplier(s) of the specific cost function(s). functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-11-08T19:13:28,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-11-08T19:13:28,055 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T19:13:28,064 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.2 set balanceSwitch=false 2024-11-08T19:13:28,082 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=379, OpenFileDescriptor=643, MaxFileDescriptor=1048576, SystemLoadAverage=407, ProcessCount=11, AvailableMemoryMB=2370 2024-11-08T19:13:28,085 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-11-08T19:13:28,085 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-11-08T19:13:28,086 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 692b295ad45c,39647,1731093203061 2024-11-08T19:13:28,086 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 692b295ad45c,39647,1731093203061: Simulated kill ***** 2024-11-08T19:13:28,086 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-08T19:13:28,088 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-08T19:13:28,090 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-08T19:13:28,091 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-08T19:13:28,091 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-08T19:13:28,091 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 298318144 }, "NonHeapMemoryUsage": { "committed": 151912448, "init": 7667712, "max": -1, "used": 149497464 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "692b295ad45c" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "692b295ad45c" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "692b295ad45c" } ] } 2024-11-08T19:13:28,094 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39509 {}] master.MasterRpcServices(700): 692b295ad45c,39647,1731093203061 reported a fatal error: ***** ABORTING region server 692b295ad45c,39647,1731093203061: Simulated kill ***** 2024-11-08T19:13:28,096 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '692b295ad45c,39647,1731093203061' ***** 2024-11-08T19:13:28,096 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-08T19:13:28,097 INFO [RS:2;692b295ad45c:39647 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T19:13:28,097 INFO [RS:2;692b295ad45c:39647 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-08T19:13:28,097 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T19:13:28,097 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T19:13:28,097 INFO [RS:2;692b295ad45c:39647 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-08T19:13:28,097 INFO [RS:2;692b295ad45c:39647 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T19:13:28,097 INFO [RS:2;692b295ad45c:39647 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;692b295ad45c:39647. 2024-11-08T19:13:28,097 DEBUG [RS:2;692b295ad45c:39647 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T19:13:28,102 DEBUG [RS:2;692b295ad45c:39647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T19:13:28,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741836_1012 (size=32114) 2024-11-08T19:13:28,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741836_1012 (size=32114) 2024-11-08T19:13:28,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741836_1012 (size=32114) 2024-11-08T19:13:28,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741834_1010 (size=2237) 2024-11-08T19:13:28,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741834_1010 (size=2237) 2024-11-08T19:13:28,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741834_1010 (size=2237) 2024-11-08T19:13:28,113 DEBUG [RS:2;692b295ad45c:39647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T19:13:28,113 INFO [RS:2;692b295ad45c:39647 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T19:13:28,113 INFO [RS:2;692b295ad45c:39647 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39647 2024-11-08T19:13:28,157 INFO [regionserver/692b295ad45c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T19:13:28,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T19:13:28,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39647-0x1011bddfb000003, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/692b295ad45c,39647,1731093203061 2024-11-08T19:13:28,209 INFO [RS:2;692b295ad45c:39647 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T19:13:28,223 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [692b295ad45c,39647,1731093203061] 2024-11-08T19:13:28,232 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/692b295ad45c,39647,1731093203061 already deleted, retry=false 2024-11-08T19:13:28,232 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 692b295ad45c,39647,1731093203061 on 692b295ad45c,39509,1731093201567 2024-11-08T19:13:28,239 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true 2024-11-08T19:13:28,244 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=57 for 692b295ad45c,39647,1731093203061 (carryingMeta=true) 692b295ad45c,39647,1731093203061/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@6f9da861[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-08T19:13:28,244 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true 2024-11-08T19:13:28,247 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true, isMeta: true 2024-11-08T19:13:28,249 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,39647,1731093203061-splitting 2024-11-08T19:13:28,250 INFO [PEWorker-1 {}] master.SplitWALManager(105): 692b295ad45c,39647,1731093203061 WAL count=1, meta=true 2024-11-08T19:13:28,253 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta}] 2024-11-08T19:13:28,259 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:28,262 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977}] 2024-11-08T19:13:28,299 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T19:13:28,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39647-0x1011bddfb000003, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T19:13:28,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39647-0x1011bddfb000003, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T19:13:28,322 INFO [RS:2;692b295ad45c:39647 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T19:13:28,323 INFO [RS:2;692b295ad45c:39647 {}] regionserver.HRegionServer(1031): Exiting; stopping=692b295ad45c,39647,1731093203061; zookeeper connection closed. 2024-11-08T19:13:28,323 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2d1bb325 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2d1bb325 2024-11-08T19:13:28,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=59 2024-11-08T19:13:28,454 INFO [RS_LOG_REPLAY_OPS-regionserver/692b295ad45c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(299): Splitting hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,39647,1731093203061-splitting/692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, size=31.4 K (32114bytes) 2024-11-08T19:13:28,454 DEBUG [RS_LOG_REPLAY_OPS-regionserver/692b295ad45c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-08T19:13:28,454 DEBUG [RS_LOG_REPLAY_OPS-regionserver/692b295ad45c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-08T19:13:28,455 INFO [RS_LOG_REPLAY_OPS-regionserver/692b295ad45c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,39647,1731093203061-splitting/692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta 2024-11-08T19:13:28,460 INFO [RS_LOG_REPLAY_OPS-regionserver/692b295ad45c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,39647,1731093203061-splitting/692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta after 3ms 2024-11-08T19:13:28,468 INFO [RS_LOG_REPLAY_OPS-regionserver/692b295ad45c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(310): Open hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,39647,1731093203061-splitting/692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta took 15ms 2024-11-08T19:13:28,479 DEBUG [RS_LOG_REPLAY_OPS-regionserver/692b295ad45c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-08T19:13:28,485 INFO [RS_LOG_REPLAY_OPS-regionserver/692b295ad45c:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta.temp 2024-11-08T19:13:28,488 DEBUG [RS_LOG_REPLAY_OPS-regionserver/692b295ad45c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,39647,1731093203061-splitting/692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta so closing down 2024-11-08T19:13:28,488 INFO [RS_LOG_REPLAY_OPS-regionserver/692b295ad45c:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta.temp 2024-11-08T19:13:28,488 DEBUG [RS_LOG_REPLAY_OPS-regionserver/692b295ad45c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-08T19:13:28,492 INFO [RS_LOG_REPLAY_OPS-regionserver/692b295ad45c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(145): 3 split writer threads finished 2024-11-08T19:13:28,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741864_1040 (size=31873) 2024-11-08T19:13:28,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741864_1040 (size=31873) 2024-11-08T19:13:28,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741864_1040 (size=31873) 2024-11-08T19:13:28,505 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta.temp (wrote 57 edits, skipped 0 edits in 2 ms) 2024-11-08T19:13:28,507 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta.temp to hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-08T19:13:28,511 INFO [RS_LOG_REPLAY_OPS-regionserver/692b295ad45c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(425): Processed 58 edits across 1 Regions in 38 ms; skipped=1; WAL=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,39647,1731093203061-splitting/692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false 2024-11-08T19:13:28,511 DEBUG [RS_LOG_REPLAY_OPS-regionserver/692b295ad45c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(428): Completed split of hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,39647,1731093203061-splitting/692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, journal: Splitting hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,39647,1731093203061-splitting/692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, size=31.4 K (32114bytes) at 1731093208454Finishing writing output for hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,39647,1731093203061-splitting/692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta so closing down at 1731093208488 (+34 ms)Creating recovered edits writer path=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta.temp at 17310932084883 split writer threads finished at 1731093208492 (+4 ms)Closed recovered edits writer path=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta.temp (wrote 57 edits, skipped 0 edits in 2 ms) at 1731093208505 (+13 ms)Rename recovered edits hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta.temp to hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 at 1731093208508 (+3 ms)Processed 58 edits across 1 Regions in 38 ms; skipped=1; WAL=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,39647,1731093203061-splitting/692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false at 1731093208511 (+3 ms) 2024-11-08T19:13:28,511 DEBUG [RS_LOG_REPLAY_OPS-regionserver/692b295ad45c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,39647,1731093203061-splitting/692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta 2024-11-08T19:13:28,512 DEBUG [RS_LOG_REPLAY_OPS-regionserver/692b295ad45c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-08T19:13:28,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-08T19:13:28,521 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,39647,1731093203061-splitting/692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta to hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/oldWALs 2024-11-08T19:13:28,524 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-08T19:13:28,524 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 in 260 msec 2024-11-08T19:13:28,527 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:28,531 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-08T19:13:28,531 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 in 275 msec 2024-11-08T19:13:28,533 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,39647,1731093203061-splitting dir is empty, no logs to split. 2024-11-08T19:13:28,533 INFO [PEWorker-1 {}] master.SplitWALManager(105): 692b295ad45c,39647,1731093203061 WAL count=0, meta=true 2024-11-08T19:13:28,533 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 692b295ad45c,39647,1731093203061 WAL splitting is done? wals=0, meta=true 2024-11-08T19:13:28,535 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-08T19:13:28,537 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T19:13:28,538 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T19:13:28,659 INFO [regionserver/692b295ad45c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-08T19:13:28,660 INFO [regionserver/692b295ad45c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-08T19:13:28,688 DEBUG [692b295ad45c:39509 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-08T19:13:28,688 DEBUG [692b295ad45c:39509 {}] balancer.BalancerClusterState(204): Hosts are {692b295ad45c=0} racks are {/default-rack=0} 2024-11-08T19:13:28,689 DEBUG [692b295ad45c:39509 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T19:13:28,689 DEBUG [692b295ad45c:39509 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T19:13:28,689 DEBUG [692b295ad45c:39509 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T19:13:28,689 DEBUG [692b295ad45c:39509 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T19:13:28,689 INFO [692b295ad45c:39509 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T19:13:28,689 INFO [692b295ad45c:39509 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T19:13:28,689 DEBUG [692b295ad45c:39509 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T19:13:28,689 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=692b295ad45c,36007,1731093202744 2024-11-08T19:13:28,691 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 692b295ad45c,36007,1731093202744, state=OPENING 2024-11-08T19:13:28,756 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:28,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:28,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:28,756 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T19:13:28,756 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:28,756 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:28,756 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:28,756 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,36007,1731093202744}] 2024-11-08T19:13:28,915 INFO [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-08T19:13:28,915 INFO [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T19:13:28,916 INFO [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-08T19:13:28,919 INFO [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=692b295ad45c%2C36007%2C1731093202744.meta, suffix=.meta, logDir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,36007,1731093202744, archiveDir=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/oldWALs, maxLogs=32 2024-11-08T19:13:28,935 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,36007,1731093202744/692b295ad45c%2C36007%2C1731093202744.meta.1731093208921.meta, exclude list is [], retry=0 2024-11-08T19:13:28,938 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46657,DS-5e1bf434-3af5-4fc9-abe7-9a902de021b6,DISK] 2024-11-08T19:13:28,939 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40347,DS-6e01471a-b8b4-43a1-88b3-b57bf9ad1710,DISK] 2024-11-08T19:13:28,939 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42249,DS-c0e88d76-cfe1-4c35-bbea-3e8b1423e597,DISK] 2024-11-08T19:13:28,948 INFO [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,36007,1731093202744/692b295ad45c%2C36007%2C1731093202744.meta.1731093208921.meta 2024-11-08T19:13:28,948 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36293:36293),(127.0.0.1/127.0.0.1:39837:39837),(127.0.0.1/127.0.0.1:46659:46659)] 2024-11-08T19:13:28,949 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-08T19:13:28,949 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-08T19:13:28,949 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-08T19:13:28,949 INFO [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-08T19:13:28,949 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-08T19:13:28,949 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:28,949 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-08T19:13:28,950 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-08T19:13:28,951 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-08T19:13:28,952 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T19:13:28,952 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:28,953 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T19:13:28,953 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-08T19:13:28,954 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T19:13:28,954 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:28,955 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T19:13:28,955 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-08T19:13:28,956 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T19:13:28,956 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:28,956 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T19:13:28,956 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-08T19:13:28,957 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T19:13:28,957 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:28,958 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T19:13:28,958 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T19:13:28,959 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/hbase/meta/1588230740 2024-11-08T19:13:28,961 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/hbase/meta/1588230740 2024-11-08T19:13:28,962 INFO [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-08T19:13:28,981 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5793): Applied 264, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=60, path=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-08T19:13:28,986 INFO [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=28 KB heapSize=48.31 KB 2024-11-08T19:13:29,070 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/hbase/meta/1588230740/.tmp/info/83661c1dea794cbeadb846692eaeb420 is 138, key is test,bbb,1731093205731.8cb2ac5239e07e076f513604c29c2ca8./info:regioninfo/1731093206597/Put/seqid=0 2024-11-08T19:13:29,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741866_1042 (size=34924) 2024-11-08T19:13:29,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741866_1042 (size=34924) 2024-11-08T19:13:29,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741866_1042 (size=34924) 2024-11-08T19:13:29,085 INFO [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.86 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/hbase/meta/1588230740/.tmp/info/83661c1dea794cbeadb846692eaeb420 2024-11-08T19:13:29,160 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/hbase/meta/1588230740/.tmp/ns/611fcf62b1974e6789e931959a5d7374 is 43, key is default/ns:d/1731093205496/Put/seqid=0 2024-11-08T19:13:29,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741867_1043 (size=5153) 2024-11-08T19:13:29,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741867_1043 (size=5153) 2024-11-08T19:13:29,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741867_1043 (size=5153) 2024-11-08T19:13:29,171 INFO [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/hbase/meta/1588230740/.tmp/ns/611fcf62b1974e6789e931959a5d7374 2024-11-08T19:13:29,201 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/hbase/meta/1588230740/.tmp/table/1abaacb859bf4404abaa73fe805db1c2 is 40, key is test/table:state/1731093206939/Put/seqid=0 2024-11-08T19:13:29,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741868_1044 (size=5165) 2024-11-08T19:13:29,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741868_1044 (size=5165) 2024-11-08T19:13:29,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741868_1044 (size=5165) 2024-11-08T19:13:29,211 INFO [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/hbase/meta/1588230740/.tmp/table/1abaacb859bf4404abaa73fe805db1c2 2024-11-08T19:13:29,220 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/hbase/meta/1588230740/.tmp/info/83661c1dea794cbeadb846692eaeb420 as hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/hbase/meta/1588230740/info/83661c1dea794cbeadb846692eaeb420 2024-11-08T19:13:29,229 INFO [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/hbase/meta/1588230740/info/83661c1dea794cbeadb846692eaeb420, entries=260, sequenceid=60, filesize=34.1 K 2024-11-08T19:13:29,232 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/hbase/meta/1588230740/.tmp/ns/611fcf62b1974e6789e931959a5d7374 as hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/hbase/meta/1588230740/ns/611fcf62b1974e6789e931959a5d7374 2024-11-08T19:13:29,240 INFO [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/hbase/meta/1588230740/ns/611fcf62b1974e6789e931959a5d7374, entries=2, sequenceid=60, filesize=5.0 K 2024-11-08T19:13:29,242 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/hbase/meta/1588230740/.tmp/table/1abaacb859bf4404abaa73fe805db1c2 as hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/hbase/meta/1588230740/table/1abaacb859bf4404abaa73fe805db1c2 2024-11-08T19:13:29,250 INFO [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/hbase/meta/1588230740/table/1abaacb859bf4404abaa73fe805db1c2, entries=2, sequenceid=60, filesize=5.0 K 2024-11-08T19:13:29,255 INFO [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 268ms, sequenceid=60, compaction requested=false; wal=null 2024-11-08T19:13:29,256 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-08T19:13:29,259 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-08T19:13:29,261 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T19:13:29,261 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T19:13:29,262 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T19:13:29,263 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T19:13:29,266 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/hbase/meta/1588230740/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=1 2024-11-08T19:13:29,267 INFO [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=61; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73662003, jitterRate=0.09764938056468964}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T19:13:29,268 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-08T19:13:29,268 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731093208950Writing region info on filesystem at 1731093208950Initializing all the Stores at 1731093208951 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731093208951Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731093208951Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731093208951Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731093208951Obtaining lock to block concurrent updates at 1731093208986 (+35 ms)Preparing flush snapshotting stores in 1588230740 at 1731093208986Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=28672, getHeapSize=49408, getOffHeapSize=0, getCellsCount=264 at 1731093208989 (+3 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731093208990 (+1 ms)Flushing 1588230740/info: creating writer at 1731093208991 (+1 ms)Flushing 1588230740/info: appending metadata at 1731093209056 (+65 ms)Flushing 1588230740/info: closing flushed file at 1731093209059 (+3 ms)Flushing 1588230740/ns: creating writer at 1731093209142 (+83 ms)Flushing 1588230740/ns: appending metadata at 1731093209159 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1731093209159Flushing 1588230740/table: creating writer at 1731093209180 (+21 ms)Flushing 1588230740/table: appending metadata at 1731093209200 (+20 ms)Flushing 1588230740/table: closing flushed file at 1731093209200Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62776092: reopening flushed file at 1731093209218 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f00b2c3: reopening flushed file at 1731093209231 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ce0f3cc: reopening flushed file at 1731093209241 (+10 ms)Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 268ms, sequenceid=60, compaction requested=false; wal=null at 1731093209255 (+14 ms)Cleaning up temporary data from old regions at 1731093209261 (+6 ms)Running coprocessor post-open hooks at 1731093209268 (+7 ms)Region opened successfully at 1731093209268 2024-11-08T19:13:29,269 INFO [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=61, masterSystemTime=1731093208909 2024-11-08T19:13:29,272 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-08T19:13:29,272 INFO [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-08T19:13:29,273 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=61, regionLocation=692b295ad45c,36007,1731093202744 2024-11-08T19:13:29,274 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 692b295ad45c,36007,1731093202744, state=OPEN 2024-11-08T19:13:29,324 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:29,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:29,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:29,325 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=61, ppid=60, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=692b295ad45c,36007,1731093202744 2024-11-08T19:13:29,325 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:29,325 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:29,325 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:29,327 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-08T19:13:29,327 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,36007,1731093202744 in 569 msec 2024-11-08T19:13:29,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=57 2024-11-08T19:13:29,330 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 792 msec 2024-11-08T19:13:29,330 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 692b295ad45c,39647,1731093203061 had 9 regions 2024-11-08T19:13:29,331 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true, isMeta: false 2024-11-08T19:13:29,333 INFO [PEWorker-1 {}] master.SplitWALManager(105): 692b295ad45c,39647,1731093203061 WAL count=1, meta=false 2024-11-08T19:13:29,333 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811}] 2024-11-08T19:13:29,335 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:29,336 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977}] 2024-11-08T19:13:29,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=63 2024-11-08T19:13:29,504 INFO [RS_LOG_REPLAY_OPS-regionserver/692b295ad45c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(299): Splitting hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,39647,1731093203061-splitting/692b295ad45c%2C39647%2C1731093203061.1731093204811, size=2.2 K (2237bytes) 2024-11-08T19:13:29,504 INFO [RS_LOG_REPLAY_OPS-regionserver/692b295ad45c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,39647,1731093203061-splitting/692b295ad45c%2C39647%2C1731093203061.1731093204811 2024-11-08T19:13:29,505 INFO [RS_LOG_REPLAY_OPS-regionserver/692b295ad45c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,39647,1731093203061-splitting/692b295ad45c%2C39647%2C1731093203061.1731093204811 after 1ms 2024-11-08T19:13:29,508 INFO [RS_LOG_REPLAY_OPS-regionserver/692b295ad45c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(310): Open hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,39647,1731093203061-splitting/692b295ad45c%2C39647%2C1731093203061.1731093204811 took 4ms 2024-11-08T19:13:29,515 DEBUG [RS_LOG_REPLAY_OPS-regionserver/692b295ad45c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 234c6774a52fc6a092eb269a4a390dc7: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-08T19:13:29,517 DEBUG [RS_LOG_REPLAY_OPS-regionserver/692b295ad45c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 8cb2ac5239e07e076f513604c29c2ca8: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-08T19:13:29,518 DEBUG [RS_LOG_REPLAY_OPS-regionserver/692b295ad45c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 22cccf3cc4c2d3343ceae07ff16f899f: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-08T19:13:29,520 DEBUG [RS_LOG_REPLAY_OPS-regionserver/692b295ad45c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for f1496a2d5d6037cbe8d2f7a8f1407184: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-08T19:13:29,522 DEBUG [RS_LOG_REPLAY_OPS-regionserver/692b295ad45c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 53b7061bfce2939f050c57221eed7f2a: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-08T19:13:29,524 DEBUG [RS_LOG_REPLAY_OPS-regionserver/692b295ad45c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 2548b6c044cd2c9ceb7dec340e2cde6c: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-08T19:13:29,526 DEBUG [RS_LOG_REPLAY_OPS-regionserver/692b295ad45c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 6e461a86c03ca80ab184602fce3cc56c: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-08T19:13:29,528 DEBUG [RS_LOG_REPLAY_OPS-regionserver/692b295ad45c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 0095b58ae56790b54bf50c65ac0754fa: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-08T19:13:29,528 DEBUG [RS_LOG_REPLAY_OPS-regionserver/692b295ad45c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,39647,1731093203061-splitting/692b295ad45c%2C39647%2C1731093203061.1731093204811 so closing down 2024-11-08T19:13:29,528 DEBUG [RS_LOG_REPLAY_OPS-regionserver/692b295ad45c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-08T19:13:29,528 INFO [RS_LOG_REPLAY_OPS-regionserver/692b295ad45c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(145): 3 split writer threads finished 2024-11-08T19:13:29,528 INFO [RS_LOG_REPLAY_OPS-regionserver/692b295ad45c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(425): Processed 8 edits across 0 Regions in 16 ms; skipped=8; WAL=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,39647,1731093203061-splitting/692b295ad45c%2C39647%2C1731093203061.1731093204811, size=2.2 K, length=2237, corrupted=false, cancelled=false 2024-11-08T19:13:29,528 DEBUG [RS_LOG_REPLAY_OPS-regionserver/692b295ad45c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(428): Completed split of hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,39647,1731093203061-splitting/692b295ad45c%2C39647%2C1731093203061.1731093204811, journal: Splitting hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,39647,1731093203061-splitting/692b295ad45c%2C39647%2C1731093203061.1731093204811, size=2.2 K (2237bytes) at 1731093209504Finishing writing output for hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,39647,1731093203061-splitting/692b295ad45c%2C39647%2C1731093203061.1731093204811 so closing down at 1731093209528 (+24 ms)3 split writer threads finished at 1731093209528Processed 8 edits across 0 Regions in 16 ms; skipped=8; WAL=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,39647,1731093203061-splitting/692b295ad45c%2C39647%2C1731093203061.1731093204811, size=2.2 K, length=2237, corrupted=false, cancelled=false at 1731093209528 2024-11-08T19:13:29,529 DEBUG [RS_LOG_REPLAY_OPS-regionserver/692b295ad45c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,39647,1731093203061-splitting/692b295ad45c%2C39647%2C1731093203061.1731093204811 2024-11-08T19:13:29,529 DEBUG [RS_LOG_REPLAY_OPS-regionserver/692b295ad45c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-08T19:13:29,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39509 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-08T19:13:29,535 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,39647,1731093203061-splitting/692b295ad45c%2C39647%2C1731093203061.1731093204811 to hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/oldWALs 2024-11-08T19:13:29,538 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-08T19:13:29,538 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 in 199 msec 2024-11-08T19:13:29,539 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:29,542 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=57 2024-11-08T19:13:29,542 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 in 207 msec 2024-11-08T19:13:29,544 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/WALs/692b295ad45c,39647,1731093203061-splitting dir is empty, no logs to split. 2024-11-08T19:13:29,545 INFO [PEWorker-1 {}] master.SplitWALManager(105): 692b295ad45c,39647,1731093203061 WAL count=0, meta=false 2024-11-08T19:13:29,545 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 692b295ad45c,39647,1731093203061 WAL splitting is done? wals=0, meta=false 2024-11-08T19:13:29,547 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN}] 2024-11-08T19:13:29,550 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN 2024-11-08T19:13:29,551 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T19:13:29,552 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN 2024-11-08T19:13:29,553 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T19:13:29,555 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN 2024-11-08T19:13:29,555 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T19:13:29,557 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN 2024-11-08T19:13:29,558 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T19:13:29,560 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN 2024-11-08T19:13:29,562 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T19:13:29,563 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN 2024-11-08T19:13:29,564 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T19:13:29,565 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN 2024-11-08T19:13:29,566 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T19:13:29,568 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN 2024-11-08T19:13:29,569 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T19:13:29,702 DEBUG [692b295ad45c:39509 {}] balancer.BalancerClusterState(204): Hosts are {692b295ad45c=0} racks are {/default-rack=0} 2024-11-08T19:13:29,702 DEBUG [692b295ad45c:39509 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T19:13:29,702 DEBUG [692b295ad45c:39509 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T19:13:29,702 DEBUG [692b295ad45c:39509 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T19:13:29,702 DEBUG [692b295ad45c:39509 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T19:13:29,702 INFO [692b295ad45c:39509 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T19:13:29,702 INFO [692b295ad45c:39509 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T19:13:29,702 DEBUG [692b295ad45c:39509 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T19:13:29,703 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=22cccf3cc4c2d3343ceae07ff16f899f, regionState=OPENING, regionLocation=692b295ad45c,36007,1731093202744 2024-11-08T19:13:29,704 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=f1496a2d5d6037cbe8d2f7a8f1407184, regionState=OPENING, regionLocation=692b295ad45c,36007,1731093202744 2024-11-08T19:13:29,704 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=6e461a86c03ca80ab184602fce3cc56c, regionState=OPENING, regionLocation=692b295ad45c,36007,1731093202744 2024-11-08T19:13:29,704 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 692b295ad45c:39647 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 692b295ad45c/172.17.0.2:39647 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T19:13:29,705 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=53b7061bfce2939f050c57221eed7f2a, regionState=OPENING, regionLocation=692b295ad45c,36007,1731093202744 2024-11-08T19:13:29,706 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=234c6774a52fc6a092eb269a4a390dc7, regionState=OPENING, regionLocation=692b295ad45c,42693,1731093202977 2024-11-08T19:13:29,706 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=692b295ad45c,39647,1731093203061, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=692b295ad45c,39647,1731093203061, seqNum=-1, error=java.net.ConnectException: Call to address=692b295ad45c:39647 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 692b295ad45c/172.17.0.2:39647 2024-11-08T19:13:29,706 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=0095b58ae56790b54bf50c65ac0754fa, regionState=OPENING, regionLocation=692b295ad45c,42693,1731093202977 2024-11-08T19:13:29,706 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=692b295ad45c,39647,1731093203061, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-08T19:13:29,707 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=692b295ad45c,39647,1731093203061, seqNum=-1 from cache 2024-11-08T19:13:29,707 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=2548b6c044cd2c9ceb7dec340e2cde6c, regionState=OPENING, regionLocation=692b295ad45c,42693,1731093202977 2024-11-08T19:13:29,707 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T19:13:29,707 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=692b295ad45c,36007,1731093202744, seqNum=-1] 2024-11-08T19:13:29,707 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T19:13:29,708 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 692b295ad45c:39647 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 692b295ad45c/172.17.0.2:39647 2024-11-08T19:13:29,708 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=8cb2ac5239e07e076f513604c29c2ca8, regionState=OPENING, regionLocation=692b295ad45c,42693,1731093202977 2024-11-08T19:13:29,708 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 692b295ad45c:39647 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 692b295ad45c/172.17.0.2:39647 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T19:13:29,709 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=692b295ad45c,39647,1731093203061, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=692b295ad45c,36007,1731093202744, seqNum=-1, error=java.net.ConnectException: Call to address=692b295ad45c:39647 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 692b295ad45c/172.17.0.2:39647 2024-11-08T19:13:29,709 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=692b295ad45c,39647,1731093203061, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=692b295ad45c,36007,1731093202744, seqNum=-1, error=java.net.ConnectException: Call to address=692b295ad45c:39647 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 692b295ad45c/172.17.0.2:39647 2024-11-08T19:13:29,709 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=692b295ad45c,39647,1731093203061, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=692b295ad45c,36007,1731093202744, seqNum=-1, error=java.net.ConnectException: Call to address=692b295ad45c:39647 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 692b295ad45c/172.17.0.2:39647 2024-11-08T19:13:29,709 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38549, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T19:13:29,709 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=692b295ad45c,39647,1731093203061, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=692b295ad45c,36007,1731093202744, seqNum=-1, error=java.net.ConnectException: Call to address=692b295ad45c:39647 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 692b295ad45c/172.17.0.2:39647 2024-11-08T19:13:29,710 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=692b295ad45c,39647,1731093203061, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=692b295ad45c,36007,1731093202744, seqNum=-1, error=java.net.ConnectException: Call to address=692b295ad45c:39647 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 692b295ad45c/172.17.0.2:39647 2024-11-08T19:13:29,710 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 692b295ad45c:39647 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 692b295ad45c/172.17.0.2:39647 2024-11-08T19:13:29,712 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN because future has completed 2024-11-08T19:13:29,712 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2548b6c044cd2c9ceb7dec340e2cde6c, server=692b295ad45c,42693,1731093202977}] 2024-11-08T19:13:29,713 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN because future has completed 2024-11-08T19:13:29,716 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977}] 2024-11-08T19:13:29,813 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN because future has completed 2024-11-08T19:13:29,813 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22cccf3cc4c2d3343ceae07ff16f899f, server=692b295ad45c,36007,1731093202744}] 2024-11-08T19:13:29,822 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN because future has completed 2024-11-08T19:13:29,823 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1496a2d5d6037cbe8d2f7a8f1407184, server=692b295ad45c,36007,1731093202744}] 2024-11-08T19:13:29,823 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN because future has completed 2024-11-08T19:13:29,825 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN because future has completed 2024-11-08T19:13:29,826 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN because future has completed 2024-11-08T19:13:29,827 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN because future has completed 2024-11-08T19:13:29,828 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0095b58ae56790b54bf50c65ac0754fa, server=692b295ad45c,42693,1731093202977}] 2024-11-08T19:13:29,831 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 234c6774a52fc6a092eb269a4a390dc7, server=692b295ad45c,42693,1731093202977}] 2024-11-08T19:13:29,834 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 53b7061bfce2939f050c57221eed7f2a, server=692b295ad45c,36007,1731093202744}] 2024-11-08T19:13:29,838 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6e461a86c03ca80ab184602fce3cc56c, server=692b295ad45c,36007,1731093202744}] 2024-11-08T19:13:29,869 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(132): Open test,bbb,1731093205731.8cb2ac5239e07e076f513604c29c2ca8. 2024-11-08T19:13:29,869 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7752): Opening region: {ENCODED => 8cb2ac5239e07e076f513604c29c2ca8, NAME => 'test,bbb,1731093205731.8cb2ac5239e07e076f513604c29c2ca8.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-08T19:13:29,870 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8cb2ac5239e07e076f513604c29c2ca8 2024-11-08T19:13:29,870 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(898): Instantiated test,bbb,1731093205731.8cb2ac5239e07e076f513604c29c2ca8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:29,870 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7794): checking encryption for 8cb2ac5239e07e076f513604c29c2ca8 2024-11-08T19:13:29,870 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7797): checking classloading for 8cb2ac5239e07e076f513604c29c2ca8 2024-11-08T19:13:29,872 INFO [StoreOpener-8cb2ac5239e07e076f513604c29c2ca8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8cb2ac5239e07e076f513604c29c2ca8 2024-11-08T19:13:29,873 INFO [StoreOpener-8cb2ac5239e07e076f513604c29c2ca8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8cb2ac5239e07e076f513604c29c2ca8 columnFamilyName family 2024-11-08T19:13:29,873 DEBUG [StoreOpener-8cb2ac5239e07e076f513604c29c2ca8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:29,874 INFO [StoreOpener-8cb2ac5239e07e076f513604c29c2ca8-1 {}] regionserver.HStore(327): Store=8cb2ac5239e07e076f513604c29c2ca8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T19:13:29,874 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1038): replaying wal for 8cb2ac5239e07e076f513604c29c2ca8 2024-11-08T19:13:29,875 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/8cb2ac5239e07e076f513604c29c2ca8 2024-11-08T19:13:29,876 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/8cb2ac5239e07e076f513604c29c2ca8 2024-11-08T19:13:29,877 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1048): stopping wal replay for 8cb2ac5239e07e076f513604c29c2ca8 2024-11-08T19:13:29,877 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1060): Cleaning up temporary data for 8cb2ac5239e07e076f513604c29c2ca8 2024-11-08T19:13:29,879 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1093): writing seq id for 8cb2ac5239e07e076f513604c29c2ca8 2024-11-08T19:13:29,880 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1114): Opened 8cb2ac5239e07e076f513604c29c2ca8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69986631, jitterRate=0.04288206994533539}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T19:13:29,880 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8cb2ac5239e07e076f513604c29c2ca8 2024-11-08T19:13:29,880 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1006): Region open journal for 8cb2ac5239e07e076f513604c29c2ca8: Running coprocessor pre-open hook at 1731093209870Writing region info on filesystem at 1731093209870Initializing all the Stores at 1731093209871 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731093209872 (+1 ms)Cleaning up temporary data from old regions at 1731093209877 (+5 ms)Running coprocessor post-open hooks at 1731093209880 (+3 ms)Region opened successfully at 1731093209880 2024-11-08T19:13:29,881 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1731093205731.8cb2ac5239e07e076f513604c29c2ca8., pid=73, masterSystemTime=1731093209864 2024-11-08T19:13:29,884 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1731093205731.8cb2ac5239e07e076f513604c29c2ca8. 2024-11-08T19:13:29,884 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(153): Opened test,bbb,1731093205731.8cb2ac5239e07e076f513604c29c2ca8. 2024-11-08T19:13:29,884 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,ooo,1731093205731.0095b58ae56790b54bf50c65ac0754fa. 2024-11-08T19:13:29,884 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => 0095b58ae56790b54bf50c65ac0754fa, NAME => 'test,ooo,1731093205731.0095b58ae56790b54bf50c65ac0754fa.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-08T19:13:29,885 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0095b58ae56790b54bf50c65ac0754fa 2024-11-08T19:13:29,885 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,ooo,1731093205731.0095b58ae56790b54bf50c65ac0754fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:29,885 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for 0095b58ae56790b54bf50c65ac0754fa 2024-11-08T19:13:29,885 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for 0095b58ae56790b54bf50c65ac0754fa 2024-11-08T19:13:29,886 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-11-08T19:13:29,886 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977 2024-11-08T19:13:29,887 INFO [StoreOpener-0095b58ae56790b54bf50c65ac0754fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0095b58ae56790b54bf50c65ac0754fa 2024-11-08T19:13:29,887 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:29,889 INFO [StoreOpener-0095b58ae56790b54bf50c65ac0754fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0095b58ae56790b54bf50c65ac0754fa columnFamilyName family 2024-11-08T19:13:29,889 DEBUG [StoreOpener-0095b58ae56790b54bf50c65ac0754fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:29,890 INFO [StoreOpener-0095b58ae56790b54bf50c65ac0754fa-1 {}] regionserver.HStore(327): Store=0095b58ae56790b54bf50c65ac0754fa/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T19:13:29,890 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for 0095b58ae56790b54bf50c65ac0754fa 2024-11-08T19:13:29,891 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/0095b58ae56790b54bf50c65ac0754fa 2024-11-08T19:13:29,892 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/0095b58ae56790b54bf50c65ac0754fa 2024-11-08T19:13:29,893 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for 0095b58ae56790b54bf50c65ac0754fa 2024-11-08T19:13:29,893 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for 0095b58ae56790b54bf50c65ac0754fa 2024-11-08T19:13:29,895 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for 0095b58ae56790b54bf50c65ac0754fa 2024-11-08T19:13:29,896 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened 0095b58ae56790b54bf50c65ac0754fa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72397439, jitterRate=0.07880590856075287}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T19:13:29,896 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0095b58ae56790b54bf50c65ac0754fa 2024-11-08T19:13:29,897 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for 0095b58ae56790b54bf50c65ac0754fa: Running coprocessor pre-open hook at 1731093209885Writing region info on filesystem at 1731093209885Initializing all the Stores at 1731093209887 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731093209887Cleaning up temporary data from old regions at 1731093209893 (+6 ms)Running coprocessor post-open hooks at 1731093209896 (+3 ms)Region opened successfully at 1731093209896 2024-11-08T19:13:29,898 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1731093205731.0095b58ae56790b54bf50c65ac0754fa., pid=76, masterSystemTime=1731093209864 2024-11-08T19:13:29,898 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 4 more 2024-11-08T19:13:29,900 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T19:13:29,900 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6e461a86c03ca80ab184602fce3cc56c, server=692b295ad45c,36007,1731093202744 2024-11-08T19:13:29,900 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:29,900 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1731093205731.0095b58ae56790b54bf50c65ac0754fa. 2024-11-08T19:13:29,900 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened test,ooo,1731093205731.0095b58ae56790b54bf50c65ac0754fa. 2024-11-08T19:13:29,900 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,uuu,1731093205731.234c6774a52fc6a092eb269a4a390dc7. 2024-11-08T19:13:29,900 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6e461a86c03ca80ab184602fce3cc56c, server=692b295ad45c,36007,1731093202744 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:29,901 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => 234c6774a52fc6a092eb269a4a390dc7, NAME => 'test,uuu,1731093205731.234c6774a52fc6a092eb269a4a390dc7.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-08T19:13:29,901 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 234c6774a52fc6a092eb269a4a390dc7 2024-11-08T19:13:29,901 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,uuu,1731093205731.234c6774a52fc6a092eb269a4a390dc7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:29,901 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for 234c6774a52fc6a092eb269a4a390dc7 2024-11-08T19:13:29,901 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for 234c6774a52fc6a092eb269a4a390dc7 2024-11-08T19:13:29,902 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T19:13:29,902 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:29,902 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T19:13:29,903 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T19:13:29,903 INFO [StoreOpener-234c6774a52fc6a092eb269a4a390dc7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 234c6774a52fc6a092eb269a4a390dc7 2024-11-08T19:13:29,903 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T19:13:29,904 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T19:13:29,904 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T19:13:29,904 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T19:13:29,904 INFO [StoreOpener-234c6774a52fc6a092eb269a4a390dc7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 234c6774a52fc6a092eb269a4a390dc7 columnFamilyName family 2024-11-08T19:13:29,904 DEBUG [StoreOpener-234c6774a52fc6a092eb269a4a390dc7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:29,904 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T19:13:29,905 INFO [StoreOpener-234c6774a52fc6a092eb269a4a390dc7-1 {}] regionserver.HStore(327): Store=234c6774a52fc6a092eb269a4a390dc7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T19:13:29,905 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for 234c6774a52fc6a092eb269a4a390dc7 2024-11-08T19:13:29,905 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/234c6774a52fc6a092eb269a4a390dc7 2024-11-08T19:13:29,907 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/234c6774a52fc6a092eb269a4a390dc7 2024-11-08T19:13:29,907 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for 234c6774a52fc6a092eb269a4a390dc7 2024-11-08T19:13:29,907 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for 234c6774a52fc6a092eb269a4a390dc7 2024-11-08T19:13:29,909 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for 234c6774a52fc6a092eb269a4a390dc7 2024-11-08T19:13:29,910 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T19:13:29,910 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened 234c6774a52fc6a092eb269a4a390dc7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62034607, jitterRate=-0.07561232149600983}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T19:13:29,911 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 234c6774a52fc6a092eb269a4a390dc7 2024-11-08T19:13:29,911 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for 234c6774a52fc6a092eb269a4a390dc7: Running coprocessor pre-open hook at 1731093209901Writing region info on filesystem at 1731093209901Initializing all the Stores at 1731093209903 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731093209903Cleaning up temporary data from old regions at 1731093209907 (+4 ms)Running coprocessor post-open hooks at 1731093209911 (+4 ms)Region opened successfully at 1731093209911 2024-11-08T19:13:29,911 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:29,911 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:29,911 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:29,911 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:29,911 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:29,911 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:29,911 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:29,911 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:29,911 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:29,911 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:29,911 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:29,911 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:29,911 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:29,911 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:29,911 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:29,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:29,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T19:13:29,912 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1731093205731.234c6774a52fc6a092eb269a4a390dc7., pid=77, masterSystemTime=1731093209864 2024-11-08T19:13:29,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T19:13:29,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:29,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:29,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T19:13:29,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T19:13:29,913 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T19:13:29,913 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39509 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:29,914 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T19:13:29,920 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true 2024-11-08T19:13:29,920 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:29,920 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:29,920 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T19:13:29,920 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,36007,1731093202744 2024-11-08T19:13:29,920 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:29,920 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:29,920 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN 2024-11-08T19:13:29,920 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN 2024-11-08T19:13:29,920 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN 2024-11-08T19:13:29,920 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN 2024-11-08T19:13:29,920 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN 2024-11-08T19:13:29,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN 2024-11-08T19:13:29,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN 2024-11-08T19:13:29,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN 2024-11-08T19:13:29,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2548b6c044cd2c9ceb7dec340e2cde6c, server=692b295ad45c,42693,1731093202977 2024-11-08T19:13:29,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977 2024-11-08T19:13:29,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22cccf3cc4c2d3343ceae07ff16f899f, server=692b295ad45c,36007,1731093202744 2024-11-08T19:13:29,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1496a2d5d6037cbe8d2f7a8f1407184, server=692b295ad45c,36007,1731093202744 2024-11-08T19:13:29,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0095b58ae56790b54bf50c65ac0754fa, server=692b295ad45c,42693,1731093202977 2024-11-08T19:13:29,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 234c6774a52fc6a092eb269a4a390dc7, server=692b295ad45c,42693,1731093202977 2024-11-08T19:13:29,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 53b7061bfce2939f050c57221eed7f2a, server=692b295ad45c,36007,1731093202744 2024-11-08T19:13:29,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6e461a86c03ca80ab184602fce3cc56c, server=692b295ad45c,36007,1731093202744 2024-11-08T19:13:29,921 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:29,921 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:29,921 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:29,921 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:29,921 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,36007,1731093202744 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:29,921 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:29,921 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:29,921 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:29,921 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN 2024-11-08T19:13:29,921 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:29,922 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN 2024-11-08T19:13:29,922 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:29,922 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN 2024-11-08T19:13:29,922 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:29,922 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN 2024-11-08T19:13:29,922 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:29,922 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN 2024-11-08T19:13:29,922 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:29,922 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN 2024-11-08T19:13:29,922 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:29,922 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN 2024-11-08T19:13:29,922 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:29,922 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN 2024-11-08T19:13:29,922 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2548b6c044cd2c9ceb7dec340e2cde6c, server=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:29,922 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:29,922 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22cccf3cc4c2d3343ceae07ff16f899f, server=692b295ad45c,36007,1731093202744 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:29,922 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1496a2d5d6037cbe8d2f7a8f1407184, server=692b295ad45c,36007,1731093202744 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:29,922 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0095b58ae56790b54bf50c65ac0754fa, server=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:29,922 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 234c6774a52fc6a092eb269a4a390dc7, server=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:29,923 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 53b7061bfce2939f050c57221eed7f2a, server=692b295ad45c,36007,1731093202744 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:29,923 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6e461a86c03ca80ab184602fce3cc56c, server=692b295ad45c,36007,1731093202744 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:29,923 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 18 msec 2024-11-08T19:13:29,925 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=61 2024-11-08T19:13:29,925 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=692b295ad45c,36007,1731093202744, table=hbase:meta, region=1588230740 2024-11-08T19:13:29,926 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 692b295ad45c,36007,1731093202744, state=OPEN 2024-11-08T19:13:29,936 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "692b295ad45c" port: 42693 start_code: 1731093202977 } transition { transition_code: OPENED region_info { region_id: 1731093205731 table_name { namespace: "default" qualifier: "test" } start_key: "uuu" end_key: "vvv" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 77 initiating_master_active_time: 1731093203324 }; retry (#0) after 1006ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-08T19:13:29,948 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:29,948 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:29,948 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:29,948 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:29,948 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:29,948 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:29,950 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN to state=OFFLINE, location=null, table=test, region=6e461a86c03ca80ab184602fce3cc56c to restore RIT 2024-11-08T19:13:29,950 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN to state=OFFLINE, location=null, table=test, region=8cb2ac5239e07e076f513604c29c2ca8 to restore RIT 2024-11-08T19:13:29,950 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN to state=OFFLINE, location=null, table=test, region=53b7061bfce2939f050c57221eed7f2a to restore RIT 2024-11-08T19:13:29,950 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN to state=OFFLINE, location=null, table=test, region=22cccf3cc4c2d3343ceae07ff16f899f to restore RIT 2024-11-08T19:13:29,951 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN to state=OFFLINE, location=null, table=test, region=0095b58ae56790b54bf50c65ac0754fa to restore RIT 2024-11-08T19:13:29,951 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN to state=OFFLINE, location=null, table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c to restore RIT 2024-11-08T19:13:29,951 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN to state=OFFLINE, location=null, table=test, region=234c6774a52fc6a092eb269a4a390dc7 to restore RIT 2024-11-08T19:13:29,951 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN to state=OFFLINE, location=null, table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184 to restore RIT 2024-11-08T19:13:29,952 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731093239952 2024-11-08T19:13:29,952 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T19:13:29,952 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:29,954 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T19:13:29,956 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T19:13:29,956 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 53b7061bfce2939f050c57221eed7f2a, server=692b295ad45c,36007,1731093202744 2024-11-08T19:13:29,956 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:29,956 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 53b7061bfce2939f050c57221eed7f2a, server=692b295ad45c,36007,1731093202744 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:29,971 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,rrr,1731093205731.f1496a2d5d6037cbe8d2f7a8f1407184. 2024-11-08T19:13:29,971 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => f1496a2d5d6037cbe8d2f7a8f1407184, NAME => 'test,rrr,1731093205731.f1496a2d5d6037cbe8d2f7a8f1407184.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-08T19:13:29,971 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f1496a2d5d6037cbe8d2f7a8f1407184 2024-11-08T19:13:29,972 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,rrr,1731093205731.f1496a2d5d6037cbe8d2f7a8f1407184.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:29,972 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for f1496a2d5d6037cbe8d2f7a8f1407184 2024-11-08T19:13:29,972 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for f1496a2d5d6037cbe8d2f7a8f1407184 2024-11-08T19:13:29,983 INFO [StoreOpener-f1496a2d5d6037cbe8d2f7a8f1407184-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f1496a2d5d6037cbe8d2f7a8f1407184 2024-11-08T19:13:29,985 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ea8472f395c484d4998844c4dcd304, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:29,986 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cb2ac5239e07e076f513604c29c2ca8, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:29,986 INFO [StoreOpener-f1496a2d5d6037cbe8d2f7a8f1407184-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1496a2d5d6037cbe8d2f7a8f1407184 columnFamilyName family 2024-11-08T19:13:29,986 DEBUG [StoreOpener-f1496a2d5d6037cbe8d2f7a8f1407184-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:29,986 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b4a23d5efe8e832c0d5cb16df3b1369, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:29,986 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8e891c0073a08913834761f455a77e8, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:29,987 INFO [StoreOpener-f1496a2d5d6037cbe8d2f7a8f1407184-1 {}] regionserver.HStore(327): Store=f1496a2d5d6037cbe8d2f7a8f1407184/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T19:13:29,987 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdc84e1c2d9bdd5f39d647a57ff91a05, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:29,987 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for f1496a2d5d6037cbe8d2f7a8f1407184 2024-11-08T19:13:29,987 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53b7061bfce2939f050c57221eed7f2a, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:29,987 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab92690881683c86166f7f23976b5188, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:29,988 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d1d9fe849fd45b830c330834b58d508, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:29,988 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/f1496a2d5d6037cbe8d2f7a8f1407184 2024-11-08T19:13:29,988 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2548b6c044cd2c9ceb7dec340e2cde6c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:29,988 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7a8c4f9e9448f30c950ddcf996ae56f, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:29,988 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d0557ff953666f2f97a7be0e2d0bc87, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:29,989 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e461a86c03ca80ab184602fce3cc56c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:29,989 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=245339f649708a4ea812249dd3f822c4, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:29,989 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/f1496a2d5d6037cbe8d2f7a8f1407184 2024-11-08T19:13:29,989 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07363d44bad100b56e1b1aba358b0ed2, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:29,990 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0095b58ae56790b54bf50c65ac0754fa, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:29,990 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for f1496a2d5d6037cbe8d2f7a8f1407184 2024-11-08T19:13:29,990 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for f1496a2d5d6037cbe8d2f7a8f1407184 2024-11-08T19:13:29,990 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e92192d7abb7a6c62f8815702d7dc38d, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:29,990 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa4304cf54e018c1a1a2071373865b5, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:29,990 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1496a2d5d6037cbe8d2f7a8f1407184, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:29,991 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77e3781e625f0b456041758476a7d6b2, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:29,991 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1918a8713e3d1c571d1188f35d75a101, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:29,991 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=234c6774a52fc6a092eb269a4a390dc7, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:29,991 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7c9c940725de9fafd49c71f7bebc530, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:29,992 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for f1496a2d5d6037cbe8d2f7a8f1407184 2024-11-08T19:13:29,992 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9e50623b9f9d6b5fc998b958dd629458, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:29,992 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48bb25f64ce0dca8172688cf9a77f57b, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:29,992 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22cccf3cc4c2d3343ceae07ff16f899f, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:29,992 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2afe1e3e862a406c7fcb359dc1b60d57, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:29,993 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T19:13:29,992 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened f1496a2d5d6037cbe8d2f7a8f1407184; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67323024, jitterRate=0.003191232681274414}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T19:13:29,993 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731093269993 2024-11-08T19:13:29,993 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f1496a2d5d6037cbe8d2f7a8f1407184 2024-11-08T19:13:29,993 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731093329993 2024-11-08T19:13:29,993 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for f1496a2d5d6037cbe8d2f7a8f1407184: Running coprocessor pre-open hook at 1731093209972Writing region info on filesystem at 1731093209972Initializing all the Stores at 1731093209983 (+11 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731093209983Cleaning up temporary data from old regions at 1731093209990 (+7 ms)Running coprocessor post-open hooks at 1731093209993 (+3 ms)Region opened successfully at 1731093209993 2024-11-08T19:13:29,993 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 41 msec 2024-11-08T19:13:29,993 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T19:13:29,993 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:29,993 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T19:13:29,993 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T19:13:29,994 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T19:13:29,994 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1731093205731.f1496a2d5d6037cbe8d2f7a8f1407184., pid=75, masterSystemTime=1731093209966 2024-11-08T19:13:29,994 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T19:13:29,994 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T19:13:29,994 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T19:13:29,995 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39509 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:29,995 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T19:13:29,996 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "692b295ad45c" port: 36007 start_code: 1731093202744 } transition { transition_code: OPENED region_info { region_id: 1731093205731 table_name { namespace: "default" qualifier: "test" } start_key: "rrr" end_key: "sss" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 75 initiating_master_active_time: 1731093203324 }; retry (#0) after 1007ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-08T19:13:30,001 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T19:13:30,001 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,001 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,001 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,001 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,001 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,001 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,001 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,001 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,001 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,001 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,002 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,002 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,002 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,002 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,002 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,002 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T19:13:30,002 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T19:13:30,002 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,002 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,002 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T19:13:30,002 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T19:13:30,003 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T19:13:30,004 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T19:13:30,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true 2024-11-08T19:13:30,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T19:13:30,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,36007,1731093202744 2024-11-08T19:13:30,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN 2024-11-08T19:13:30,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN 2024-11-08T19:13:30,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN 2024-11-08T19:13:30,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN 2024-11-08T19:13:30,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN 2024-11-08T19:13:30,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN 2024-11-08T19:13:30,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN 2024-11-08T19:13:30,009 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN 2024-11-08T19:13:30,009 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2548b6c044cd2c9ceb7dec340e2cde6c, server=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,009 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,009 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22cccf3cc4c2d3343ceae07ff16f899f, server=692b295ad45c,36007,1731093202744 2024-11-08T19:13:30,009 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1496a2d5d6037cbe8d2f7a8f1407184, server=692b295ad45c,36007,1731093202744 2024-11-08T19:13:30,009 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0095b58ae56790b54bf50c65ac0754fa, server=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,009 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 234c6774a52fc6a092eb269a4a390dc7, server=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,009 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 53b7061bfce2939f050c57221eed7f2a, server=692b295ad45c,36007,1731093202744 2024-11-08T19:13:30,009 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,009 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,009 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,009 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,009 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,36007,1731093202744 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,009 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,009 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,009 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,009 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN 2024-11-08T19:13:30,009 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,010 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN 2024-11-08T19:13:30,010 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,010 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN 2024-11-08T19:13:30,010 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,010 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN 2024-11-08T19:13:30,010 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,010 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN 2024-11-08T19:13:30,010 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,010 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN 2024-11-08T19:13:30,010 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,010 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN 2024-11-08T19:13:30,010 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,010 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN 2024-11-08T19:13:30,010 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2548b6c044cd2c9ceb7dec340e2cde6c, server=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,010 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,010 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22cccf3cc4c2d3343ceae07ff16f899f, server=692b295ad45c,36007,1731093202744 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,010 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1496a2d5d6037cbe8d2f7a8f1407184, server=692b295ad45c,36007,1731093202744 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,010 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0095b58ae56790b54bf50c65ac0754fa, server=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,010 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 234c6774a52fc6a092eb269a4a390dc7, server=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,010 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 53b7061bfce2939f050c57221eed7f2a, server=692b295ad45c,36007,1731093202744 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,011 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 15 msec 2024-11-08T19:13:30,013 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=61 2024-11-08T19:13:30,013 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=692b295ad45c,36007,1731093202744, table=hbase:meta, region=1588230740 2024-11-08T19:13:30,013 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 692b295ad45c,36007,1731093202744, state=OPEN 2024-11-08T19:13:30,072 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,073 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,073 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,073 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,074 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN to state=OFFLINE, location=null, table=test, region=6e461a86c03ca80ab184602fce3cc56c to restore RIT 2024-11-08T19:13:30,074 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN to state=OFFLINE, location=null, table=test, region=8cb2ac5239e07e076f513604c29c2ca8 to restore RIT 2024-11-08T19:13:30,074 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN to state=OFFLINE, location=null, table=test, region=53b7061bfce2939f050c57221eed7f2a to restore RIT 2024-11-08T19:13:30,074 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN to state=OFFLINE, location=null, table=test, region=22cccf3cc4c2d3343ceae07ff16f899f to restore RIT 2024-11-08T19:13:30,074 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN to state=OFFLINE, location=null, table=test, region=0095b58ae56790b54bf50c65ac0754fa to restore RIT 2024-11-08T19:13:30,074 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN to state=OFFLINE, location=null, table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c to restore RIT 2024-11-08T19:13:30,074 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN to state=OFFLINE, location=null, table=test, region=234c6774a52fc6a092eb269a4a390dc7 to restore RIT 2024-11-08T19:13:30,074 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN to state=OFFLINE, location=null, table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184 to restore RIT 2024-11-08T19:13:30,076 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731093240075 2024-11-08T19:13:30,076 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T19:13:30,076 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,078 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T19:13:30,086 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T19:13:30,086 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 234c6774a52fc6a092eb269a4a390dc7, server=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:30,086 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 234c6774a52fc6a092eb269a4a390dc7, server=692b295ad45c,42693,1731093202977 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,087 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ea8472f395c484d4998844c4dcd304, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,088 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cb2ac5239e07e076f513604c29c2ca8, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,088 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b4a23d5efe8e832c0d5cb16df3b1369, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,088 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8e891c0073a08913834761f455a77e8, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,088 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdc84e1c2d9bdd5f39d647a57ff91a05, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,089 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53b7061bfce2939f050c57221eed7f2a, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,089 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab92690881683c86166f7f23976b5188, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,089 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d1d9fe849fd45b830c330834b58d508, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,089 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2548b6c044cd2c9ceb7dec340e2cde6c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7a8c4f9e9448f30c950ddcf996ae56f, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d0557ff953666f2f97a7be0e2d0bc87, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e461a86c03ca80ab184602fce3cc56c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=245339f649708a4ea812249dd3f822c4, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07363d44bad100b56e1b1aba358b0ed2, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0095b58ae56790b54bf50c65ac0754fa, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e92192d7abb7a6c62f8815702d7dc38d, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa4304cf54e018c1a1a2071373865b5, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1496a2d5d6037cbe8d2f7a8f1407184, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77e3781e625f0b456041758476a7d6b2, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1918a8713e3d1c571d1188f35d75a101, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=234c6774a52fc6a092eb269a4a390dc7, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7c9c940725de9fafd49c71f7bebc530, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9e50623b9f9d6b5fc998b958dd629458, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48bb25f64ce0dca8172688cf9a77f57b, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22cccf3cc4c2d3343ceae07ff16f899f, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2afe1e3e862a406c7fcb359dc1b60d57, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,093 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T19:13:30,093 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731093270093 2024-11-08T19:13:30,093 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731093330093 2024-11-08T19:13:30,093 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 17 msec 2024-11-08T19:13:30,093 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T19:13:30,094 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:30,094 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T19:13:30,094 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T19:13:30,094 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T19:13:30,094 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T19:13:30,094 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T19:13:30,095 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T19:13:30,096 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T19:13:30,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T19:13:30,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,103 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,103 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,103 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,103 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,103 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,103 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T19:13:30,103 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T19:13:30,103 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,103 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,103 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T19:13:30,103 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T19:13:30,103 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T19:13:30,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T19:13:30,108 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true 2024-11-08T19:13:30,108 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,108 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,108 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T19:13:30,108 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,36007,1731093202744 2024-11-08T19:13:30,108 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,109 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,109 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN 2024-11-08T19:13:30,109 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN 2024-11-08T19:13:30,109 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN 2024-11-08T19:13:30,109 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN 2024-11-08T19:13:30,109 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN 2024-11-08T19:13:30,109 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN 2024-11-08T19:13:30,109 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN 2024-11-08T19:13:30,109 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN 2024-11-08T19:13:30,109 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2548b6c044cd2c9ceb7dec340e2cde6c, server=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,109 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,109 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22cccf3cc4c2d3343ceae07ff16f899f, server=692b295ad45c,36007,1731093202744 2024-11-08T19:13:30,109 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1496a2d5d6037cbe8d2f7a8f1407184, server=692b295ad45c,36007,1731093202744 2024-11-08T19:13:30,109 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0095b58ae56790b54bf50c65ac0754fa, server=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,110 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 234c6774a52fc6a092eb269a4a390dc7, server=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,110 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,110 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,110 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,110 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,110 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,36007,1731093202744 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,110 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,110 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,110 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,110 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN 2024-11-08T19:13:30,110 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,110 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN 2024-11-08T19:13:30,110 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,110 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN 2024-11-08T19:13:30,110 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,111 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN 2024-11-08T19:13:30,111 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,111 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN 2024-11-08T19:13:30,111 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,111 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN 2024-11-08T19:13:30,111 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,111 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN 2024-11-08T19:13:30,111 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,111 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN 2024-11-08T19:13:30,111 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2548b6c044cd2c9ceb7dec340e2cde6c, server=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,111 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,111 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22cccf3cc4c2d3343ceae07ff16f899f, server=692b295ad45c,36007,1731093202744 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,111 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1496a2d5d6037cbe8d2f7a8f1407184, server=692b295ad45c,36007,1731093202744 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,111 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0095b58ae56790b54bf50c65ac0754fa, server=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,111 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 234c6774a52fc6a092eb269a4a390dc7, server=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,112 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 15 msec 2024-11-08T19:13:30,113 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=61 2024-11-08T19:13:30,113 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=692b295ad45c,36007,1731093202744, table=hbase:meta, region=1588230740 2024-11-08T19:13:30,114 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 692b295ad45c,36007,1731093202744, state=OPEN 2024-11-08T19:13:30,127 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,127 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,127 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,127 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,127 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,127 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,128 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN to state=OFFLINE, location=null, table=test, region=6e461a86c03ca80ab184602fce3cc56c to restore RIT 2024-11-08T19:13:30,128 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN to state=OFFLINE, location=null, table=test, region=8cb2ac5239e07e076f513604c29c2ca8 to restore RIT 2024-11-08T19:13:30,128 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN to state=OFFLINE, location=null, table=test, region=53b7061bfce2939f050c57221eed7f2a to restore RIT 2024-11-08T19:13:30,128 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN to state=OFFLINE, location=null, table=test, region=22cccf3cc4c2d3343ceae07ff16f899f to restore RIT 2024-11-08T19:13:30,128 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN to state=OFFLINE, location=null, table=test, region=0095b58ae56790b54bf50c65ac0754fa to restore RIT 2024-11-08T19:13:30,128 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN to state=OFFLINE, location=null, table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c to restore RIT 2024-11-08T19:13:30,128 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN to state=OFFLINE, location=null, table=test, region=234c6774a52fc6a092eb269a4a390dc7 to restore RIT 2024-11-08T19:13:30,128 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN to state=OFFLINE, location=null, table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184 to restore RIT 2024-11-08T19:13:30,129 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731093240129 2024-11-08T19:13:30,129 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T19:13:30,129 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,130 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T19:13:30,132 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T19:13:30,132 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0095b58ae56790b54bf50c65ac0754fa, server=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,132 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:30,132 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0095b58ae56790b54bf50c65ac0754fa, server=692b295ad45c,42693,1731093202977 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,138 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ea8472f395c484d4998844c4dcd304, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,138 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cb2ac5239e07e076f513604c29c2ca8, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,138 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b4a23d5efe8e832c0d5cb16df3b1369, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,138 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8e891c0073a08913834761f455a77e8, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,139 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdc84e1c2d9bdd5f39d647a57ff91a05, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,139 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53b7061bfce2939f050c57221eed7f2a, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,139 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab92690881683c86166f7f23976b5188, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,139 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d1d9fe849fd45b830c330834b58d508, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,140 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2548b6c044cd2c9ceb7dec340e2cde6c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,140 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7a8c4f9e9448f30c950ddcf996ae56f, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,140 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d0557ff953666f2f97a7be0e2d0bc87, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,140 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e461a86c03ca80ab184602fce3cc56c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,140 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=245339f649708a4ea812249dd3f822c4, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,141 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07363d44bad100b56e1b1aba358b0ed2, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,141 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0095b58ae56790b54bf50c65ac0754fa, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,141 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e92192d7abb7a6c62f8815702d7dc38d, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,141 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa4304cf54e018c1a1a2071373865b5, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,142 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1496a2d5d6037cbe8d2f7a8f1407184, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,142 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77e3781e625f0b456041758476a7d6b2, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,142 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1918a8713e3d1c571d1188f35d75a101, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,142 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=234c6774a52fc6a092eb269a4a390dc7, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,142 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7c9c940725de9fafd49c71f7bebc530, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,143 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9e50623b9f9d6b5fc998b958dd629458, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,143 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48bb25f64ce0dca8172688cf9a77f57b, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,143 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22cccf3cc4c2d3343ceae07ff16f899f, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,143 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2afe1e3e862a406c7fcb359dc1b60d57, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,143 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T19:13:30,143 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731093270143 2024-11-08T19:13:30,144 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731093330144 2024-11-08T19:13:30,144 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 14 msec 2024-11-08T19:13:30,144 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T19:13:30,144 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:30,144 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T19:13:30,144 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T19:13:30,144 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T19:13:30,144 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T19:13:30,144 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T19:13:30,145 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T19:13:30,145 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T19:13:30,150 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T19:13:30,151 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,151 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,151 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,151 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,151 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,151 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,151 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,151 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,151 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,151 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,151 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,151 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,151 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,151 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T19:13:30,151 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T19:13:30,151 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,151 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,151 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T19:13:30,151 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T19:13:30,152 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T19:13:30,153 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T19:13:30,156 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true 2024-11-08T19:13:30,156 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,156 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,156 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T19:13:30,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,36007,1731093202744 2024-11-08T19:13:30,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN 2024-11-08T19:13:30,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN 2024-11-08T19:13:30,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN 2024-11-08T19:13:30,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN 2024-11-08T19:13:30,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN 2024-11-08T19:13:30,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN 2024-11-08T19:13:30,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN 2024-11-08T19:13:30,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN 2024-11-08T19:13:30,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2548b6c044cd2c9ceb7dec340e2cde6c, server=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22cccf3cc4c2d3343ceae07ff16f899f, server=692b295ad45c,36007,1731093202744 2024-11-08T19:13:30,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1496a2d5d6037cbe8d2f7a8f1407184, server=692b295ad45c,36007,1731093202744 2024-11-08T19:13:30,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0095b58ae56790b54bf50c65ac0754fa, server=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,157 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,157 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,158 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,158 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,158 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,36007,1731093202744 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,158 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,158 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,158 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,158 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN 2024-11-08T19:13:30,158 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,158 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN 2024-11-08T19:13:30,158 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,158 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN 2024-11-08T19:13:30,158 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,158 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN 2024-11-08T19:13:30,158 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,158 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN 2024-11-08T19:13:30,158 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,158 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN 2024-11-08T19:13:30,158 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,158 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN 2024-11-08T19:13:30,158 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,159 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN 2024-11-08T19:13:30,159 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2548b6c044cd2c9ceb7dec340e2cde6c, server=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,159 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,159 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22cccf3cc4c2d3343ceae07ff16f899f, server=692b295ad45c,36007,1731093202744 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,159 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1496a2d5d6037cbe8d2f7a8f1407184, server=692b295ad45c,36007,1731093202744 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,159 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0095b58ae56790b54bf50c65ac0754fa, server=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,159 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 13 msec 2024-11-08T19:13:30,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=61 2024-11-08T19:13:30,161 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=692b295ad45c,36007,1731093202744, table=hbase:meta, region=1588230740 2024-11-08T19:13:30,161 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 692b295ad45c,36007,1731093202744, state=OPEN 2024-11-08T19:13:30,169 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,169 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,169 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,169 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,169 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,169 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,170 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN to state=OFFLINE, location=null, table=test, region=6e461a86c03ca80ab184602fce3cc56c to restore RIT 2024-11-08T19:13:30,170 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN to state=OFFLINE, location=null, table=test, region=8cb2ac5239e07e076f513604c29c2ca8 to restore RIT 2024-11-08T19:13:30,170 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN to state=OFFLINE, location=null, table=test, region=53b7061bfce2939f050c57221eed7f2a to restore RIT 2024-11-08T19:13:30,170 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN to state=OFFLINE, location=null, table=test, region=22cccf3cc4c2d3343ceae07ff16f899f to restore RIT 2024-11-08T19:13:30,170 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN to state=OFFLINE, location=null, table=test, region=0095b58ae56790b54bf50c65ac0754fa to restore RIT 2024-11-08T19:13:30,170 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN to state=OFFLINE, location=null, table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c to restore RIT 2024-11-08T19:13:30,170 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN to state=OFFLINE, location=null, table=test, region=234c6774a52fc6a092eb269a4a390dc7 to restore RIT 2024-11-08T19:13:30,170 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN to state=OFFLINE, location=null, table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184 to restore RIT 2024-11-08T19:13:30,171 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731093240171 2024-11-08T19:13:30,171 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T19:13:30,171 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,172 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T19:13:30,178 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T19:13:30,178 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1496a2d5d6037cbe8d2f7a8f1407184, server=692b295ad45c,36007,1731093202744 2024-11-08T19:13:30,178 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:30,178 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1496a2d5d6037cbe8d2f7a8f1407184, server=692b295ad45c,36007,1731093202744 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,179 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ea8472f395c484d4998844c4dcd304, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,179 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cb2ac5239e07e076f513604c29c2ca8, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,180 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b4a23d5efe8e832c0d5cb16df3b1369, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,180 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8e891c0073a08913834761f455a77e8, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,180 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdc84e1c2d9bdd5f39d647a57ff91a05, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,180 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53b7061bfce2939f050c57221eed7f2a, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,180 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab92690881683c86166f7f23976b5188, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,181 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d1d9fe849fd45b830c330834b58d508, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,181 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2548b6c044cd2c9ceb7dec340e2cde6c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,181 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7a8c4f9e9448f30c950ddcf996ae56f, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,181 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d0557ff953666f2f97a7be0e2d0bc87, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,181 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e461a86c03ca80ab184602fce3cc56c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,182 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=245339f649708a4ea812249dd3f822c4, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,182 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07363d44bad100b56e1b1aba358b0ed2, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,182 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0095b58ae56790b54bf50c65ac0754fa, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,182 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e92192d7abb7a6c62f8815702d7dc38d, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,183 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa4304cf54e018c1a1a2071373865b5, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,183 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1496a2d5d6037cbe8d2f7a8f1407184, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,183 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77e3781e625f0b456041758476a7d6b2, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,183 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1918a8713e3d1c571d1188f35d75a101, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,183 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=234c6774a52fc6a092eb269a4a390dc7, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7c9c940725de9fafd49c71f7bebc530, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9e50623b9f9d6b5fc998b958dd629458, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48bb25f64ce0dca8172688cf9a77f57b, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22cccf3cc4c2d3343ceae07ff16f899f, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2afe1e3e862a406c7fcb359dc1b60d57, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,184 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T19:13:30,184 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731093270184 2024-11-08T19:13:30,185 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731093330184 2024-11-08T19:13:30,185 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 13 msec 2024-11-08T19:13:30,185 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T19:13:30,185 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:30,185 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T19:13:30,185 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T19:13:30,185 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T19:13:30,185 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T19:13:30,185 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T19:13:30,186 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T19:13:30,186 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T19:13:30,189 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39509 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,190 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T19:13:30,190 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,190 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,190 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,190 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,190 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,190 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,191 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,191 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,191 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,191 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,191 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,191 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,191 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T19:13:30,191 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T19:13:30,191 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,191 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,191 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T19:13:30,191 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T19:13:30,191 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T19:13:30,192 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T19:13:30,195 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true 2024-11-08T19:13:30,195 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,195 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,195 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T19:13:30,195 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,36007,1731093202744 2024-11-08T19:13:30,195 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,195 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,195 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN 2024-11-08T19:13:30,196 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN 2024-11-08T19:13:30,196 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN 2024-11-08T19:13:30,196 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN 2024-11-08T19:13:30,196 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN 2024-11-08T19:13:30,196 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN 2024-11-08T19:13:30,196 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN 2024-11-08T19:13:30,196 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN 2024-11-08T19:13:30,196 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2548b6c044cd2c9ceb7dec340e2cde6c, server=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,196 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,196 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22cccf3cc4c2d3343ceae07ff16f899f, server=692b295ad45c,36007,1731093202744 2024-11-08T19:13:30,196 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1496a2d5d6037cbe8d2f7a8f1407184, server=692b295ad45c,36007,1731093202744 2024-11-08T19:13:30,196 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,196 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,196 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,196 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,196 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,36007,1731093202744 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,196 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,196 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,196 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,197 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN 2024-11-08T19:13:30,197 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,197 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN 2024-11-08T19:13:30,197 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,197 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN 2024-11-08T19:13:30,197 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,197 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN 2024-11-08T19:13:30,197 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,197 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN 2024-11-08T19:13:30,197 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,197 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN 2024-11-08T19:13:30,197 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,197 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN 2024-11-08T19:13:30,197 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,197 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN 2024-11-08T19:13:30,197 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2548b6c044cd2c9ceb7dec340e2cde6c, server=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,197 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,197 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22cccf3cc4c2d3343ceae07ff16f899f, server=692b295ad45c,36007,1731093202744 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,197 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1496a2d5d6037cbe8d2f7a8f1407184, server=692b295ad45c,36007,1731093202744 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,198 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-11-08T19:13:30,199 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=61 2024-11-08T19:13:30,199 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=692b295ad45c,36007,1731093202744, table=hbase:meta, region=1588230740 2024-11-08T19:13:30,199 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 692b295ad45c,36007,1731093202744, state=OPEN 2024-11-08T19:13:30,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,211 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,211 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,211 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,211 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,212 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN to state=OFFLINE, location=null, table=test, region=6e461a86c03ca80ab184602fce3cc56c to restore RIT 2024-11-08T19:13:30,212 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN to state=OFFLINE, location=null, table=test, region=8cb2ac5239e07e076f513604c29c2ca8 to restore RIT 2024-11-08T19:13:30,212 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN to state=OFFLINE, location=null, table=test, region=53b7061bfce2939f050c57221eed7f2a to restore RIT 2024-11-08T19:13:30,212 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN to state=OFFLINE, location=null, table=test, region=22cccf3cc4c2d3343ceae07ff16f899f to restore RIT 2024-11-08T19:13:30,212 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN to state=OFFLINE, location=null, table=test, region=0095b58ae56790b54bf50c65ac0754fa to restore RIT 2024-11-08T19:13:30,212 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN to state=OFFLINE, location=null, table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c to restore RIT 2024-11-08T19:13:30,212 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN to state=OFFLINE, location=null, table=test, region=234c6774a52fc6a092eb269a4a390dc7 to restore RIT 2024-11-08T19:13:30,212 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN to state=OFFLINE, location=null, table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184 to restore RIT 2024-11-08T19:13:30,212 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731093240212 2024-11-08T19:13:30,212 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T19:13:30,212 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,214 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T19:13:30,216 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T19:13:30,216 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22cccf3cc4c2d3343ceae07ff16f899f, server=692b295ad45c,36007,1731093202744 2024-11-08T19:13:30,216 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:30,216 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22cccf3cc4c2d3343ceae07ff16f899f, server=692b295ad45c,36007,1731093202744 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ea8472f395c484d4998844c4dcd304, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cb2ac5239e07e076f513604c29c2ca8, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b4a23d5efe8e832c0d5cb16df3b1369, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8e891c0073a08913834761f455a77e8, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdc84e1c2d9bdd5f39d647a57ff91a05, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53b7061bfce2939f050c57221eed7f2a, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab92690881683c86166f7f23976b5188, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d1d9fe849fd45b830c330834b58d508, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2548b6c044cd2c9ceb7dec340e2cde6c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7a8c4f9e9448f30c950ddcf996ae56f, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d0557ff953666f2f97a7be0e2d0bc87, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e461a86c03ca80ab184602fce3cc56c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=245339f649708a4ea812249dd3f822c4, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07363d44bad100b56e1b1aba358b0ed2, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0095b58ae56790b54bf50c65ac0754fa, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e92192d7abb7a6c62f8815702d7dc38d, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa4304cf54e018c1a1a2071373865b5, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1496a2d5d6037cbe8d2f7a8f1407184, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77e3781e625f0b456041758476a7d6b2, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1918a8713e3d1c571d1188f35d75a101, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=234c6774a52fc6a092eb269a4a390dc7, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7c9c940725de9fafd49c71f7bebc530, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9e50623b9f9d6b5fc998b958dd629458, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48bb25f64ce0dca8172688cf9a77f57b, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22cccf3cc4c2d3343ceae07ff16f899f, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2afe1e3e862a406c7fcb359dc1b60d57, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,224 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T19:13:30,224 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731093270224 2024-11-08T19:13:30,224 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731093330224 2024-11-08T19:13:30,224 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 11 msec 2024-11-08T19:13:30,224 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T19:13:30,225 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:30,225 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T19:13:30,225 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T19:13:30,225 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T19:13:30,225 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T19:13:30,225 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T19:13:30,226 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T19:13:30,226 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T19:13:30,229 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T19:13:30,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T19:13:30,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T19:13:30,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T19:13:30,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T19:13:30,231 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T19:13:30,232 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T19:13:30,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true 2024-11-08T19:13:30,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T19:13:30,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,36007,1731093202744 2024-11-08T19:13:30,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN 2024-11-08T19:13:30,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN 2024-11-08T19:13:30,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN 2024-11-08T19:13:30,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN 2024-11-08T19:13:30,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN 2024-11-08T19:13:30,235 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN 2024-11-08T19:13:30,235 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN 2024-11-08T19:13:30,235 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN 2024-11-08T19:13:30,235 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2548b6c044cd2c9ceb7dec340e2cde6c, server=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,235 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,235 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22cccf3cc4c2d3343ceae07ff16f899f, server=692b295ad45c,36007,1731093202744 2024-11-08T19:13:30,235 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,235 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,235 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,235 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,235 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,36007,1731093202744 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,235 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,235 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,235 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,235 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN 2024-11-08T19:13:30,235 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,235 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN 2024-11-08T19:13:30,236 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,236 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN 2024-11-08T19:13:30,236 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,236 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN 2024-11-08T19:13:30,236 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,236 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN 2024-11-08T19:13:30,236 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,236 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN 2024-11-08T19:13:30,236 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,236 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN 2024-11-08T19:13:30,236 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,236 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN 2024-11-08T19:13:30,236 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2548b6c044cd2c9ceb7dec340e2cde6c, server=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,236 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,236 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22cccf3cc4c2d3343ceae07ff16f899f, server=692b295ad45c,36007,1731093202744 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,236 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-08T19:13:30,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=61 2024-11-08T19:13:30,237 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=692b295ad45c,36007,1731093202744, table=hbase:meta, region=1588230740 2024-11-08T19:13:30,238 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 692b295ad45c,36007,1731093202744, state=OPEN 2024-11-08T19:13:30,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,251 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,251 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,251 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,251 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,251 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN to state=OFFLINE, location=null, table=test, region=6e461a86c03ca80ab184602fce3cc56c to restore RIT 2024-11-08T19:13:30,251 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN to state=OFFLINE, location=null, table=test, region=8cb2ac5239e07e076f513604c29c2ca8 to restore RIT 2024-11-08T19:13:30,252 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN to state=OFFLINE, location=null, table=test, region=53b7061bfce2939f050c57221eed7f2a to restore RIT 2024-11-08T19:13:30,252 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN to state=OFFLINE, location=null, table=test, region=22cccf3cc4c2d3343ceae07ff16f899f to restore RIT 2024-11-08T19:13:30,252 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN to state=OFFLINE, location=null, table=test, region=0095b58ae56790b54bf50c65ac0754fa to restore RIT 2024-11-08T19:13:30,252 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN to state=OFFLINE, location=null, table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c to restore RIT 2024-11-08T19:13:30,252 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN to state=OFFLINE, location=null, table=test, region=234c6774a52fc6a092eb269a4a390dc7 to restore RIT 2024-11-08T19:13:30,252 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN to state=OFFLINE, location=null, table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184 to restore RIT 2024-11-08T19:13:30,252 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731093240252 2024-11-08T19:13:30,252 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T19:13:30,252 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,254 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T19:13:30,255 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T19:13:30,255 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,255 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:30,255 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,259 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ea8472f395c484d4998844c4dcd304, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,259 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cb2ac5239e07e076f513604c29c2ca8, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,259 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b4a23d5efe8e832c0d5cb16df3b1369, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,259 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8e891c0073a08913834761f455a77e8, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,259 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdc84e1c2d9bdd5f39d647a57ff91a05, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,260 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53b7061bfce2939f050c57221eed7f2a, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,260 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab92690881683c86166f7f23976b5188, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,260 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d1d9fe849fd45b830c330834b58d508, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,260 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2548b6c044cd2c9ceb7dec340e2cde6c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,260 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7a8c4f9e9448f30c950ddcf996ae56f, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,260 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d0557ff953666f2f97a7be0e2d0bc87, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,261 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e461a86c03ca80ab184602fce3cc56c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,261 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=245339f649708a4ea812249dd3f822c4, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,261 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07363d44bad100b56e1b1aba358b0ed2, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,261 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0095b58ae56790b54bf50c65ac0754fa, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,261 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e92192d7abb7a6c62f8815702d7dc38d, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,261 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa4304cf54e018c1a1a2071373865b5, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,261 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1496a2d5d6037cbe8d2f7a8f1407184, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,262 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77e3781e625f0b456041758476a7d6b2, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,262 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1918a8713e3d1c571d1188f35d75a101, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,262 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=234c6774a52fc6a092eb269a4a390dc7, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,262 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7c9c940725de9fafd49c71f7bebc530, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,262 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9e50623b9f9d6b5fc998b958dd629458, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,262 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48bb25f64ce0dca8172688cf9a77f57b, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,262 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22cccf3cc4c2d3343ceae07ff16f899f, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,263 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2afe1e3e862a406c7fcb359dc1b60d57, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,263 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T19:13:30,263 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731093270263 2024-11-08T19:13:30,263 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731093330263 2024-11-08T19:13:30,263 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-08T19:13:30,263 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T19:13:30,263 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:30,263 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T19:13:30,263 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T19:13:30,263 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T19:13:30,263 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T19:13:30,264 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T19:13:30,264 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T19:13:30,265 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T19:13:30,267 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T19:13:30,268 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,268 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,268 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,268 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,268 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,268 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,268 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,268 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,268 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,268 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,268 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T19:13:30,268 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T19:13:30,268 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,268 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,268 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T19:13:30,268 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T19:13:30,269 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T19:13:30,269 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T19:13:30,271 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true 2024-11-08T19:13:30,271 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,271 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,271 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T19:13:30,271 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,36007,1731093202744 2024-11-08T19:13:30,271 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN 2024-11-08T19:13:30,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN 2024-11-08T19:13:30,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN 2024-11-08T19:13:30,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN 2024-11-08T19:13:30,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN 2024-11-08T19:13:30,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN 2024-11-08T19:13:30,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN 2024-11-08T19:13:30,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN 2024-11-08T19:13:30,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2548b6c044cd2c9ceb7dec340e2cde6c, server=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,272 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,272 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,272 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,272 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,272 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,36007,1731093202744 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,272 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,272 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,273 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,273 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN 2024-11-08T19:13:30,273 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,273 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN 2024-11-08T19:13:30,273 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,273 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN 2024-11-08T19:13:30,273 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,273 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN 2024-11-08T19:13:30,273 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,273 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN 2024-11-08T19:13:30,273 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,273 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN 2024-11-08T19:13:30,273 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,273 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN 2024-11-08T19:13:30,273 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,273 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN 2024-11-08T19:13:30,273 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2548b6c044cd2c9ceb7dec340e2cde6c, server=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,273 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,274 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-08T19:13:30,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=61 2024-11-08T19:13:30,275 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=692b295ad45c,36007,1731093202744, table=hbase:meta, region=1588230740 2024-11-08T19:13:30,275 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 692b295ad45c,36007,1731093202744, state=OPEN 2024-11-08T19:13:30,282 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39509 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,284 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,285 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,285 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,285 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,285 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN to state=OFFLINE, location=null, table=test, region=6e461a86c03ca80ab184602fce3cc56c to restore RIT 2024-11-08T19:13:30,286 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN to state=OFFLINE, location=null, table=test, region=8cb2ac5239e07e076f513604c29c2ca8 to restore RIT 2024-11-08T19:13:30,286 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN to state=OFFLINE, location=null, table=test, region=53b7061bfce2939f050c57221eed7f2a to restore RIT 2024-11-08T19:13:30,286 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN to state=OFFLINE, location=null, table=test, region=22cccf3cc4c2d3343ceae07ff16f899f to restore RIT 2024-11-08T19:13:30,286 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN to state=OFFLINE, location=null, table=test, region=0095b58ae56790b54bf50c65ac0754fa to restore RIT 2024-11-08T19:13:30,286 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN to state=OFFLINE, location=null, table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c to restore RIT 2024-11-08T19:13:30,286 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN to state=OFFLINE, location=null, table=test, region=234c6774a52fc6a092eb269a4a390dc7 to restore RIT 2024-11-08T19:13:30,286 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN to state=OFFLINE, location=null, table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184 to restore RIT 2024-11-08T19:13:30,286 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731093240286 2024-11-08T19:13:30,286 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T19:13:30,286 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,288 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T19:13:30,289 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T19:13:30,289 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2548b6c044cd2c9ceb7dec340e2cde6c, server=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:30,289 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2548b6c044cd2c9ceb7dec340e2cde6c, server=692b295ad45c,42693,1731093202977 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,293 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ea8472f395c484d4998844c4dcd304, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,294 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cb2ac5239e07e076f513604c29c2ca8, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,294 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b4a23d5efe8e832c0d5cb16df3b1369, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,294 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8e891c0073a08913834761f455a77e8, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,294 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdc84e1c2d9bdd5f39d647a57ff91a05, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,294 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53b7061bfce2939f050c57221eed7f2a, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,294 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab92690881683c86166f7f23976b5188, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,294 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d1d9fe849fd45b830c330834b58d508, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,295 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2548b6c044cd2c9ceb7dec340e2cde6c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,295 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7a8c4f9e9448f30c950ddcf996ae56f, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,295 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d0557ff953666f2f97a7be0e2d0bc87, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,295 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e461a86c03ca80ab184602fce3cc56c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,295 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=245339f649708a4ea812249dd3f822c4, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07363d44bad100b56e1b1aba358b0ed2, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0095b58ae56790b54bf50c65ac0754fa, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e92192d7abb7a6c62f8815702d7dc38d, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa4304cf54e018c1a1a2071373865b5, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1496a2d5d6037cbe8d2f7a8f1407184, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77e3781e625f0b456041758476a7d6b2, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,297 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1918a8713e3d1c571d1188f35d75a101, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,297 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=234c6774a52fc6a092eb269a4a390dc7, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,297 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7c9c940725de9fafd49c71f7bebc530, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,297 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9e50623b9f9d6b5fc998b958dd629458, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,297 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48bb25f64ce0dca8172688cf9a77f57b, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,297 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22cccf3cc4c2d3343ceae07ff16f899f, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,298 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2afe1e3e862a406c7fcb359dc1b60d57, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,298 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T19:13:30,298 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731093270298 2024-11-08T19:13:30,298 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731093330298 2024-11-08T19:13:30,298 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 11 msec 2024-11-08T19:13:30,298 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T19:13:30,298 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:30,298 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T19:13:30,298 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T19:13:30,299 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T19:13:30,299 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T19:13:30,299 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T19:13:30,299 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T19:13:30,300 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T19:13:30,303 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T19:13:30,303 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,303 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,303 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,303 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,303 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,303 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,303 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,303 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,303 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,303 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T19:13:30,303 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T19:13:30,303 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,304 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,304 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T19:13:30,304 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T19:13:30,304 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T19:13:30,305 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T19:13:30,306 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true 2024-11-08T19:13:30,306 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,306 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,306 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T19:13:30,306 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,36007,1731093202744 2024-11-08T19:13:30,306 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,306 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,307 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN 2024-11-08T19:13:30,307 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN 2024-11-08T19:13:30,307 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN 2024-11-08T19:13:30,307 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN 2024-11-08T19:13:30,307 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN 2024-11-08T19:13:30,307 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN 2024-11-08T19:13:30,307 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN 2024-11-08T19:13:30,307 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN 2024-11-08T19:13:30,307 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2548b6c044cd2c9ceb7dec340e2cde6c, server=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,307 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,307 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,307 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,307 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,307 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,36007,1731093202744 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,307 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,307 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,307 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,307 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN 2024-11-08T19:13:30,307 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,308 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN 2024-11-08T19:13:30,308 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,308 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN 2024-11-08T19:13:30,308 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,308 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN 2024-11-08T19:13:30,308 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,308 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN 2024-11-08T19:13:30,308 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,308 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN 2024-11-08T19:13:30,308 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,308 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN 2024-11-08T19:13:30,308 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,308 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN 2024-11-08T19:13:30,308 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2548b6c044cd2c9ceb7dec340e2cde6c, server=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,308 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-08T19:13:30,309 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=61 2024-11-08T19:13:30,309 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=692b295ad45c,36007,1731093202744, table=hbase:meta, region=1588230740 2024-11-08T19:13:30,310 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 692b295ad45c,36007,1731093202744, state=OPEN 2024-11-08T19:13:30,316 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,316 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,316 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,316 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,316 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,316 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,317 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN to state=OFFLINE, location=null, table=test, region=6e461a86c03ca80ab184602fce3cc56c to restore RIT 2024-11-08T19:13:30,317 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN to state=OFFLINE, location=null, table=test, region=8cb2ac5239e07e076f513604c29c2ca8 to restore RIT 2024-11-08T19:13:30,317 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN to state=OFFLINE, location=null, table=test, region=53b7061bfce2939f050c57221eed7f2a to restore RIT 2024-11-08T19:13:30,317 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN to state=OFFLINE, location=null, table=test, region=22cccf3cc4c2d3343ceae07ff16f899f to restore RIT 2024-11-08T19:13:30,317 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN to state=OFFLINE, location=null, table=test, region=0095b58ae56790b54bf50c65ac0754fa to restore RIT 2024-11-08T19:13:30,317 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN to state=OFFLINE, location=null, table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c to restore RIT 2024-11-08T19:13:30,317 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN to state=OFFLINE, location=null, table=test, region=234c6774a52fc6a092eb269a4a390dc7 to restore RIT 2024-11-08T19:13:30,317 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN to state=OFFLINE, location=null, table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184 to restore RIT 2024-11-08T19:13:30,318 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731093240318 2024-11-08T19:13:30,318 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T19:13:30,318 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,319 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T19:13:30,320 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T19:13:30,320 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN 2024-11-08T19:13:30,320 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:30,321 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,324 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ea8472f395c484d4998844c4dcd304, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,324 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cb2ac5239e07e076f513604c29c2ca8, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,324 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b4a23d5efe8e832c0d5cb16df3b1369, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,324 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8e891c0073a08913834761f455a77e8, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,324 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdc84e1c2d9bdd5f39d647a57ff91a05, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53b7061bfce2939f050c57221eed7f2a, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab92690881683c86166f7f23976b5188, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d1d9fe849fd45b830c330834b58d508, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2548b6c044cd2c9ceb7dec340e2cde6c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7a8c4f9e9448f30c950ddcf996ae56f, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d0557ff953666f2f97a7be0e2d0bc87, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,326 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e461a86c03ca80ab184602fce3cc56c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,326 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=245339f649708a4ea812249dd3f822c4, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,326 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07363d44bad100b56e1b1aba358b0ed2, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,326 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0095b58ae56790b54bf50c65ac0754fa, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,326 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e92192d7abb7a6c62f8815702d7dc38d, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,326 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa4304cf54e018c1a1a2071373865b5, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,326 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1496a2d5d6037cbe8d2f7a8f1407184, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,327 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77e3781e625f0b456041758476a7d6b2, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,327 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1918a8713e3d1c571d1188f35d75a101, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,327 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=234c6774a52fc6a092eb269a4a390dc7, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,327 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7c9c940725de9fafd49c71f7bebc530, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,327 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9e50623b9f9d6b5fc998b958dd629458, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,327 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48bb25f64ce0dca8172688cf9a77f57b, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,328 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22cccf3cc4c2d3343ceae07ff16f899f, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,328 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2afe1e3e862a406c7fcb359dc1b60d57, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,328 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T19:13:30,328 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731093270328 2024-11-08T19:13:30,328 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731093330328 2024-11-08T19:13:30,328 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-08T19:13:30,328 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T19:13:30,328 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:30,328 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T19:13:30,328 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T19:13:30,329 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T19:13:30,329 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T19:13:30,329 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T19:13:30,329 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T19:13:30,330 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T19:13:30,332 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T19:13:30,332 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,332 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,332 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,332 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,332 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,333 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,333 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,333 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,333 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T19:13:30,333 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T19:13:30,333 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,333 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,333 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T19:13:30,333 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T19:13:30,333 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T19:13:30,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T19:13:30,336 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true 2024-11-08T19:13:30,336 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,336 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,336 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T19:13:30,336 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,36007,1731093202744 2024-11-08T19:13:30,336 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,336 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,336 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN 2024-11-08T19:13:30,336 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN 2024-11-08T19:13:30,336 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN 2024-11-08T19:13:30,336 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN 2024-11-08T19:13:30,336 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN 2024-11-08T19:13:30,336 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN 2024-11-08T19:13:30,336 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN 2024-11-08T19:13:30,336 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN 2024-11-08T19:13:30,336 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,337 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,337 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,337 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,337 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,36007,1731093202744 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,337 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,337 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,337 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,337 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN 2024-11-08T19:13:30,337 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,337 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN 2024-11-08T19:13:30,337 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,337 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN 2024-11-08T19:13:30,337 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,337 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN 2024-11-08T19:13:30,337 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,337 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN 2024-11-08T19:13:30,337 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,337 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN 2024-11-08T19:13:30,337 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,338 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN 2024-11-08T19:13:30,338 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,338 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN 2024-11-08T19:13:30,338 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-08T19:13:30,339 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=61 2024-11-08T19:13:30,339 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=692b295ad45c,36007,1731093202744, table=hbase:meta, region=1588230740 2024-11-08T19:13:30,339 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 692b295ad45c,36007,1731093202744, state=OPEN 2024-11-08T19:13:30,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,348 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,348 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,348 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,348 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,348 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN to state=OFFLINE, location=null, table=test, region=6e461a86c03ca80ab184602fce3cc56c to restore RIT 2024-11-08T19:13:30,348 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN to state=OFFLINE, location=null, table=test, region=8cb2ac5239e07e076f513604c29c2ca8 to restore RIT 2024-11-08T19:13:30,349 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN to state=OFFLINE, location=null, table=test, region=53b7061bfce2939f050c57221eed7f2a to restore RIT 2024-11-08T19:13:30,349 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN to state=OFFLINE, location=null, table=test, region=22cccf3cc4c2d3343ceae07ff16f899f to restore RIT 2024-11-08T19:13:30,349 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN to state=OFFLINE, location=null, table=test, region=0095b58ae56790b54bf50c65ac0754fa to restore RIT 2024-11-08T19:13:30,349 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN to state=OFFLINE, location=null, table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c to restore RIT 2024-11-08T19:13:30,349 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN to state=OFFLINE, location=null, table=test, region=234c6774a52fc6a092eb269a4a390dc7 to restore RIT 2024-11-08T19:13:30,349 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184, ASSIGN to state=OFFLINE, location=null, table=test, region=f1496a2d5d6037cbe8d2f7a8f1407184 to restore RIT 2024-11-08T19:13:30,349 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731093240349 2024-11-08T19:13:30,349 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T19:13:30,349 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,350 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T19:13:30,352 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T19:13:30,352 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN 2024-11-08T19:13:30,352 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:30,352 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,355 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ea8472f395c484d4998844c4dcd304, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,355 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cb2ac5239e07e076f513604c29c2ca8, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,355 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b4a23d5efe8e832c0d5cb16df3b1369, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,356 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8e891c0073a08913834761f455a77e8, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,356 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdc84e1c2d9bdd5f39d647a57ff91a05, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,356 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53b7061bfce2939f050c57221eed7f2a, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,356 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab92690881683c86166f7f23976b5188, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,356 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d1d9fe849fd45b830c330834b58d508, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,356 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2548b6c044cd2c9ceb7dec340e2cde6c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,357 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7a8c4f9e9448f30c950ddcf996ae56f, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,357 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d0557ff953666f2f97a7be0e2d0bc87, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,357 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e461a86c03ca80ab184602fce3cc56c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,357 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=245339f649708a4ea812249dd3f822c4, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,357 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07363d44bad100b56e1b1aba358b0ed2, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,357 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0095b58ae56790b54bf50c65ac0754fa, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,358 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e92192d7abb7a6c62f8815702d7dc38d, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,358 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa4304cf54e018c1a1a2071373865b5, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,358 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1496a2d5d6037cbe8d2f7a8f1407184, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,358 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77e3781e625f0b456041758476a7d6b2, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,358 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1918a8713e3d1c571d1188f35d75a101, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,358 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=234c6774a52fc6a092eb269a4a390dc7, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,359 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7c9c940725de9fafd49c71f7bebc530, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,359 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9e50623b9f9d6b5fc998b958dd629458, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,359 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48bb25f64ce0dca8172688cf9a77f57b, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,359 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22cccf3cc4c2d3343ceae07ff16f899f, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,359 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2afe1e3e862a406c7fcb359dc1b60d57, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,359 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T19:13:30,359 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731093270359 2024-11-08T19:13:30,359 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731093330359 2024-11-08T19:13:30,360 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-08T19:13:30,360 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T19:13:30,360 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:30,360 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T19:13:30,360 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T19:13:30,360 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T19:13:30,360 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T19:13:30,360 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T19:13:30,360 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T19:13:30,361 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T19:13:30,364 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T19:13:30,364 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,364 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,364 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,364 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,364 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,364 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,364 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,364 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T19:13:30,364 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T19:13:30,364 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,364 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,364 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T19:13:30,364 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T19:13:30,364 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T19:13:30,365 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T19:13:30,367 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true 2024-11-08T19:13:30,367 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,367 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,367 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T19:13:30,367 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,36007,1731093202744 2024-11-08T19:13:30,367 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,367 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,367 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN 2024-11-08T19:13:30,367 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN 2024-11-08T19:13:30,367 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN 2024-11-08T19:13:30,367 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN 2024-11-08T19:13:30,367 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN 2024-11-08T19:13:30,367 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN 2024-11-08T19:13:30,367 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN 2024-11-08T19:13:30,367 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,368 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,368 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,368 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,368 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,36007,1731093202744 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,368 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,368 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,368 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,368 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN 2024-11-08T19:13:30,368 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,368 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN 2024-11-08T19:13:30,368 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,368 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN 2024-11-08T19:13:30,368 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,368 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN 2024-11-08T19:13:30,368 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,368 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN 2024-11-08T19:13:30,368 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,368 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN 2024-11-08T19:13:30,368 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,368 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN 2024-11-08T19:13:30,369 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-08T19:13:30,370 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=61 2024-11-08T19:13:30,370 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=692b295ad45c,36007,1731093202744, table=hbase:meta, region=1588230740 2024-11-08T19:13:30,370 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 692b295ad45c,36007,1731093202744, state=OPEN 2024-11-08T19:13:30,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,379 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,379 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,380 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,380 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,380 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN to state=OFFLINE, location=null, table=test, region=6e461a86c03ca80ab184602fce3cc56c to restore RIT 2024-11-08T19:13:30,380 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN to state=OFFLINE, location=null, table=test, region=8cb2ac5239e07e076f513604c29c2ca8 to restore RIT 2024-11-08T19:13:30,380 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN to state=OFFLINE, location=null, table=test, region=53b7061bfce2939f050c57221eed7f2a to restore RIT 2024-11-08T19:13:30,380 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN to state=OFFLINE, location=null, table=test, region=22cccf3cc4c2d3343ceae07ff16f899f to restore RIT 2024-11-08T19:13:30,380 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN to state=OFFLINE, location=null, table=test, region=0095b58ae56790b54bf50c65ac0754fa to restore RIT 2024-11-08T19:13:30,380 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN to state=OFFLINE, location=null, table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c to restore RIT 2024-11-08T19:13:30,380 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=234c6774a52fc6a092eb269a4a390dc7, ASSIGN to state=OFFLINE, location=null, table=test, region=234c6774a52fc6a092eb269a4a390dc7 to restore RIT 2024-11-08T19:13:30,381 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731093240381 2024-11-08T19:13:30,381 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T19:13:30,381 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,382 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T19:13:30,383 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T19:13:30,383 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN 2024-11-08T19:13:30,383 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:30,383 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,386 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ea8472f395c484d4998844c4dcd304, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,387 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cb2ac5239e07e076f513604c29c2ca8, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,387 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b4a23d5efe8e832c0d5cb16df3b1369, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,387 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8e891c0073a08913834761f455a77e8, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,387 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdc84e1c2d9bdd5f39d647a57ff91a05, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,387 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53b7061bfce2939f050c57221eed7f2a, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,387 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab92690881683c86166f7f23976b5188, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,388 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d1d9fe849fd45b830c330834b58d508, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,388 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2548b6c044cd2c9ceb7dec340e2cde6c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,388 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7a8c4f9e9448f30c950ddcf996ae56f, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,388 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d0557ff953666f2f97a7be0e2d0bc87, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,388 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e461a86c03ca80ab184602fce3cc56c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,388 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=245339f649708a4ea812249dd3f822c4, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,388 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07363d44bad100b56e1b1aba358b0ed2, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,388 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0095b58ae56790b54bf50c65ac0754fa, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,388 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e92192d7abb7a6c62f8815702d7dc38d, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa4304cf54e018c1a1a2071373865b5, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1496a2d5d6037cbe8d2f7a8f1407184, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77e3781e625f0b456041758476a7d6b2, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1918a8713e3d1c571d1188f35d75a101, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=234c6774a52fc6a092eb269a4a390dc7, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7c9c940725de9fafd49c71f7bebc530, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9e50623b9f9d6b5fc998b958dd629458, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48bb25f64ce0dca8172688cf9a77f57b, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22cccf3cc4c2d3343ceae07ff16f899f, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,390 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2afe1e3e862a406c7fcb359dc1b60d57, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,390 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T19:13:30,390 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731093270390 2024-11-08T19:13:30,390 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731093330390 2024-11-08T19:13:30,390 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-08T19:13:30,390 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T19:13:30,390 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:30,390 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T19:13:30,390 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T19:13:30,390 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T19:13:30,390 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T19:13:30,390 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T19:13:30,391 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T19:13:30,391 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T19:13:30,393 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T19:13:30,393 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,393 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,393 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,393 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,393 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,393 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,393 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T19:13:30,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T19:13:30,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T19:13:30,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T19:13:30,394 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T19:13:30,395 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T19:13:30,395 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39509 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,396 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true 2024-11-08T19:13:30,396 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,396 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,396 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T19:13:30,396 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,36007,1731093202744 2024-11-08T19:13:30,396 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,396 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,396 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN 2024-11-08T19:13:30,396 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN 2024-11-08T19:13:30,396 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN 2024-11-08T19:13:30,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN 2024-11-08T19:13:30,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN 2024-11-08T19:13:30,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN 2024-11-08T19:13:30,397 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,397 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,397 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,397 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,397 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,36007,1731093202744 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,397 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,397 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,397 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,397 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN 2024-11-08T19:13:30,397 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,397 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN 2024-11-08T19:13:30,397 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,398 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN 2024-11-08T19:13:30,398 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,398 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN 2024-11-08T19:13:30,398 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,398 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN 2024-11-08T19:13:30,398 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,398 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN 2024-11-08T19:13:30,398 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-08T19:13:30,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=61 2024-11-08T19:13:30,399 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=692b295ad45c,36007,1731093202744, table=hbase:meta, region=1588230740 2024-11-08T19:13:30,399 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 692b295ad45c,36007,1731093202744, state=OPEN 2024-11-08T19:13:30,411 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,411 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,411 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,411 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,412 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN to state=OFFLINE, location=null, table=test, region=6e461a86c03ca80ab184602fce3cc56c to restore RIT 2024-11-08T19:13:30,412 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN to state=OFFLINE, location=null, table=test, region=8cb2ac5239e07e076f513604c29c2ca8 to restore RIT 2024-11-08T19:13:30,412 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN to state=OFFLINE, location=null, table=test, region=53b7061bfce2939f050c57221eed7f2a to restore RIT 2024-11-08T19:13:30,412 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN to state=OFFLINE, location=null, table=test, region=22cccf3cc4c2d3343ceae07ff16f899f to restore RIT 2024-11-08T19:13:30,412 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN to state=OFFLINE, location=null, table=test, region=0095b58ae56790b54bf50c65ac0754fa to restore RIT 2024-11-08T19:13:30,412 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c, ASSIGN to state=OFFLINE, location=null, table=test, region=2548b6c044cd2c9ceb7dec340e2cde6c to restore RIT 2024-11-08T19:13:30,413 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731093240413 2024-11-08T19:13:30,413 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T19:13:30,413 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,414 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T19:13:30,415 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T19:13:30,415 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN 2024-11-08T19:13:30,415 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:30,415 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ea8472f395c484d4998844c4dcd304, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cb2ac5239e07e076f513604c29c2ca8, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b4a23d5efe8e832c0d5cb16df3b1369, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8e891c0073a08913834761f455a77e8, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdc84e1c2d9bdd5f39d647a57ff91a05, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53b7061bfce2939f050c57221eed7f2a, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,420 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab92690881683c86166f7f23976b5188, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,420 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d1d9fe849fd45b830c330834b58d508, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,420 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2548b6c044cd2c9ceb7dec340e2cde6c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,420 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7a8c4f9e9448f30c950ddcf996ae56f, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,420 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d0557ff953666f2f97a7be0e2d0bc87, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,420 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e461a86c03ca80ab184602fce3cc56c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,420 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=245339f649708a4ea812249dd3f822c4, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,420 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07363d44bad100b56e1b1aba358b0ed2, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,421 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0095b58ae56790b54bf50c65ac0754fa, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,421 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e92192d7abb7a6c62f8815702d7dc38d, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,421 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa4304cf54e018c1a1a2071373865b5, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,421 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1496a2d5d6037cbe8d2f7a8f1407184, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,421 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77e3781e625f0b456041758476a7d6b2, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,421 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1918a8713e3d1c571d1188f35d75a101, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,421 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=234c6774a52fc6a092eb269a4a390dc7, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,422 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7c9c940725de9fafd49c71f7bebc530, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,422 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9e50623b9f9d6b5fc998b958dd629458, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,422 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48bb25f64ce0dca8172688cf9a77f57b, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,422 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22cccf3cc4c2d3343ceae07ff16f899f, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,422 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2afe1e3e862a406c7fcb359dc1b60d57, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,422 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T19:13:30,422 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731093270422 2024-11-08T19:13:30,422 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731093330422 2024-11-08T19:13:30,422 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-08T19:13:30,422 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T19:13:30,422 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:30,423 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T19:13:30,423 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T19:13:30,423 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T19:13:30,423 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T19:13:30,423 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T19:13:30,423 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T19:13:30,424 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T19:13:30,426 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T19:13:30,426 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,426 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,426 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,426 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,426 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,427 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T19:13:30,427 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T19:13:30,427 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,427 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,427 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T19:13:30,427 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T19:13:30,427 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T19:13:30,428 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T19:13:30,429 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true 2024-11-08T19:13:30,429 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,429 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,429 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T19:13:30,429 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,36007,1731093202744 2024-11-08T19:13:30,429 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,429 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,429 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN 2024-11-08T19:13:30,429 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN 2024-11-08T19:13:30,429 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN 2024-11-08T19:13:30,429 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN 2024-11-08T19:13:30,429 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN 2024-11-08T19:13:30,430 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,430 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,430 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,430 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,430 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,36007,1731093202744 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,430 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,430 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,430 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,430 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN 2024-11-08T19:13:30,430 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,430 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN 2024-11-08T19:13:30,430 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,430 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN 2024-11-08T19:13:30,430 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,430 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN 2024-11-08T19:13:30,430 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,430 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN 2024-11-08T19:13:30,430 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-08T19:13:30,431 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=61 2024-11-08T19:13:30,431 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=692b295ad45c,36007,1731093202744, table=hbase:meta, region=1588230740 2024-11-08T19:13:30,432 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 692b295ad45c,36007,1731093202744, state=OPEN 2024-11-08T19:13:30,442 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,443 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,443 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,443 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,443 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN to state=OFFLINE, location=null, table=test, region=6e461a86c03ca80ab184602fce3cc56c to restore RIT 2024-11-08T19:13:30,443 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN to state=OFFLINE, location=null, table=test, region=8cb2ac5239e07e076f513604c29c2ca8 to restore RIT 2024-11-08T19:13:30,443 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN to state=OFFLINE, location=null, table=test, region=53b7061bfce2939f050c57221eed7f2a to restore RIT 2024-11-08T19:13:30,444 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN to state=OFFLINE, location=null, table=test, region=22cccf3cc4c2d3343ceae07ff16f899f to restore RIT 2024-11-08T19:13:30,444 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0095b58ae56790b54bf50c65ac0754fa, ASSIGN to state=OFFLINE, location=null, table=test, region=0095b58ae56790b54bf50c65ac0754fa to restore RIT 2024-11-08T19:13:30,444 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731093240444 2024-11-08T19:13:30,444 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T19:13:30,444 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,445 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T19:13:30,446 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T19:13:30,446 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN 2024-11-08T19:13:30,446 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:30,446 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,449 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ea8472f395c484d4998844c4dcd304, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,449 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cb2ac5239e07e076f513604c29c2ca8, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,449 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b4a23d5efe8e832c0d5cb16df3b1369, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,449 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8e891c0073a08913834761f455a77e8, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,449 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdc84e1c2d9bdd5f39d647a57ff91a05, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,449 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53b7061bfce2939f050c57221eed7f2a, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab92690881683c86166f7f23976b5188, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d1d9fe849fd45b830c330834b58d508, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2548b6c044cd2c9ceb7dec340e2cde6c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7a8c4f9e9448f30c950ddcf996ae56f, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d0557ff953666f2f97a7be0e2d0bc87, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e461a86c03ca80ab184602fce3cc56c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=245339f649708a4ea812249dd3f822c4, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07363d44bad100b56e1b1aba358b0ed2, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0095b58ae56790b54bf50c65ac0754fa, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e92192d7abb7a6c62f8815702d7dc38d, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa4304cf54e018c1a1a2071373865b5, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1496a2d5d6037cbe8d2f7a8f1407184, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77e3781e625f0b456041758476a7d6b2, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1918a8713e3d1c571d1188f35d75a101, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=234c6774a52fc6a092eb269a4a390dc7, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7c9c940725de9fafd49c71f7bebc530, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9e50623b9f9d6b5fc998b958dd629458, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48bb25f64ce0dca8172688cf9a77f57b, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22cccf3cc4c2d3343ceae07ff16f899f, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2afe1e3e862a406c7fcb359dc1b60d57, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,451 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T19:13:30,451 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731093270451 2024-11-08T19:13:30,451 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731093330451 2024-11-08T19:13:30,451 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-08T19:13:30,451 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T19:13:30,451 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:30,452 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T19:13:30,452 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T19:13:30,452 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T19:13:30,452 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T19:13:30,452 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T19:13:30,452 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T19:13:30,453 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T19:13:30,454 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T19:13:30,454 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,455 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,455 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,455 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,455 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T19:13:30,455 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T19:13:30,455 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,455 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,455 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T19:13:30,455 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T19:13:30,455 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T19:13:30,455 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T19:13:30,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true 2024-11-08T19:13:30,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T19:13:30,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,36007,1731093202744 2024-11-08T19:13:30,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN 2024-11-08T19:13:30,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN 2024-11-08T19:13:30,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN 2024-11-08T19:13:30,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN 2024-11-08T19:13:30,456 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,457 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,457 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,457 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,457 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,36007,1731093202744 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,457 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,457 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,457 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,457 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN 2024-11-08T19:13:30,457 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,457 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN 2024-11-08T19:13:30,457 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,457 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN 2024-11-08T19:13:30,457 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,457 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN 2024-11-08T19:13:30,457 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-08T19:13:30,458 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=61 2024-11-08T19:13:30,458 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=692b295ad45c,36007,1731093202744, table=hbase:meta, region=1588230740 2024-11-08T19:13:30,458 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 692b295ad45c,36007,1731093202744, state=OPEN 2024-11-08T19:13:30,463 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,463 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,463 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,463 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,463 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,464 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,464 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN to state=OFFLINE, location=null, table=test, region=6e461a86c03ca80ab184602fce3cc56c to restore RIT 2024-11-08T19:13:30,464 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN to state=OFFLINE, location=null, table=test, region=8cb2ac5239e07e076f513604c29c2ca8 to restore RIT 2024-11-08T19:13:30,464 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN to state=OFFLINE, location=null, table=test, region=53b7061bfce2939f050c57221eed7f2a to restore RIT 2024-11-08T19:13:30,464 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22cccf3cc4c2d3343ceae07ff16f899f, ASSIGN to state=OFFLINE, location=null, table=test, region=22cccf3cc4c2d3343ceae07ff16f899f to restore RIT 2024-11-08T19:13:30,465 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731093240465 2024-11-08T19:13:30,465 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T19:13:30,465 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,466 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T19:13:30,467 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T19:13:30,467 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN 2024-11-08T19:13:30,467 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:30,467 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ea8472f395c484d4998844c4dcd304, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cb2ac5239e07e076f513604c29c2ca8, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b4a23d5efe8e832c0d5cb16df3b1369, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8e891c0073a08913834761f455a77e8, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdc84e1c2d9bdd5f39d647a57ff91a05, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53b7061bfce2939f050c57221eed7f2a, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab92690881683c86166f7f23976b5188, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d1d9fe849fd45b830c330834b58d508, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2548b6c044cd2c9ceb7dec340e2cde6c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7a8c4f9e9448f30c950ddcf996ae56f, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d0557ff953666f2f97a7be0e2d0bc87, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e461a86c03ca80ab184602fce3cc56c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=245339f649708a4ea812249dd3f822c4, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07363d44bad100b56e1b1aba358b0ed2, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0095b58ae56790b54bf50c65ac0754fa, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e92192d7abb7a6c62f8815702d7dc38d, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa4304cf54e018c1a1a2071373865b5, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1496a2d5d6037cbe8d2f7a8f1407184, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77e3781e625f0b456041758476a7d6b2, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1918a8713e3d1c571d1188f35d75a101, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=234c6774a52fc6a092eb269a4a390dc7, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7c9c940725de9fafd49c71f7bebc530, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9e50623b9f9d6b5fc998b958dd629458, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48bb25f64ce0dca8172688cf9a77f57b, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22cccf3cc4c2d3343ceae07ff16f899f, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2afe1e3e862a406c7fcb359dc1b60d57, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,473 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T19:13:30,473 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731093270473 2024-11-08T19:13:30,473 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731093330473 2024-11-08T19:13:30,473 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-08T19:13:30,473 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T19:13:30,473 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:30,473 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T19:13:30,473 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T19:13:30,473 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T19:13:30,473 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T19:13:30,473 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T19:13:30,473 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T19:13:30,474 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T19:13:30,475 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T19:13:30,475 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,475 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,475 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,475 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T19:13:30,475 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T19:13:30,476 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,476 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,476 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T19:13:30,476 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T19:13:30,476 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T19:13:30,476 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T19:13:30,477 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true 2024-11-08T19:13:30,477 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,477 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,477 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T19:13:30,477 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,36007,1731093202744 2024-11-08T19:13:30,477 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,477 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,477 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN 2024-11-08T19:13:30,477 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN 2024-11-08T19:13:30,477 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN 2024-11-08T19:13:30,477 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,477 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,477 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,477 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,477 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,36007,1731093202744 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,477 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,477 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,477 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,477 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN 2024-11-08T19:13:30,478 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,478 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN 2024-11-08T19:13:30,478 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,478 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN 2024-11-08T19:13:30,478 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-08T19:13:30,478 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=61 2024-11-08T19:13:30,478 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=692b295ad45c,36007,1731093202744, table=hbase:meta, region=1588230740 2024-11-08T19:13:30,479 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 692b295ad45c,36007,1731093202744, state=OPEN 2024-11-08T19:13:30,484 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,485 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,485 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,485 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,485 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN to state=OFFLINE, location=null, table=test, region=6e461a86c03ca80ab184602fce3cc56c to restore RIT 2024-11-08T19:13:30,485 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN to state=OFFLINE, location=null, table=test, region=8cb2ac5239e07e076f513604c29c2ca8 to restore RIT 2024-11-08T19:13:30,485 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=53b7061bfce2939f050c57221eed7f2a, ASSIGN to state=OFFLINE, location=null, table=test, region=53b7061bfce2939f050c57221eed7f2a to restore RIT 2024-11-08T19:13:30,486 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731093240486 2024-11-08T19:13:30,486 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T19:13:30,486 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,487 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T19:13:30,488 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T19:13:30,488 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN 2024-11-08T19:13:30,488 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:30,488 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ea8472f395c484d4998844c4dcd304, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cb2ac5239e07e076f513604c29c2ca8, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b4a23d5efe8e832c0d5cb16df3b1369, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8e891c0073a08913834761f455a77e8, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdc84e1c2d9bdd5f39d647a57ff91a05, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53b7061bfce2939f050c57221eed7f2a, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,493 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab92690881683c86166f7f23976b5188, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,493 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d1d9fe849fd45b830c330834b58d508, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,493 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2548b6c044cd2c9ceb7dec340e2cde6c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,493 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7a8c4f9e9448f30c950ddcf996ae56f, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,493 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d0557ff953666f2f97a7be0e2d0bc87, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,493 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e461a86c03ca80ab184602fce3cc56c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,494 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=245339f649708a4ea812249dd3f822c4, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,494 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07363d44bad100b56e1b1aba358b0ed2, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,494 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0095b58ae56790b54bf50c65ac0754fa, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,494 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e92192d7abb7a6c62f8815702d7dc38d, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,494 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa4304cf54e018c1a1a2071373865b5, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,494 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1496a2d5d6037cbe8d2f7a8f1407184, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,494 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77e3781e625f0b456041758476a7d6b2, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,494 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1918a8713e3d1c571d1188f35d75a101, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,495 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=234c6774a52fc6a092eb269a4a390dc7, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,495 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7c9c940725de9fafd49c71f7bebc530, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,495 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9e50623b9f9d6b5fc998b958dd629458, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,495 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48bb25f64ce0dca8172688cf9a77f57b, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,495 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22cccf3cc4c2d3343ceae07ff16f899f, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,495 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2afe1e3e862a406c7fcb359dc1b60d57, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,495 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T19:13:30,495 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731093270495 2024-11-08T19:13:30,495 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731093330495 2024-11-08T19:13:30,496 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-08T19:13:30,496 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T19:13:30,496 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:30,496 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T19:13:30,496 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T19:13:30,496 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T19:13:30,496 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T19:13:30,496 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T19:13:30,496 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T19:13:30,497 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T19:13:30,498 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39509 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,499 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T19:13:30,499 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,500 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,500 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T19:13:30,500 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T19:13:30,500 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,500 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,500 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T19:13:30,500 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T19:13:30,500 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T19:13:30,501 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T19:13:30,502 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true 2024-11-08T19:13:30,502 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,502 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,502 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T19:13:30,502 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,36007,1731093202744 2024-11-08T19:13:30,503 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,503 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,503 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN 2024-11-08T19:13:30,503 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN 2024-11-08T19:13:30,503 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,503 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,503 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,503 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,503 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,36007,1731093202744 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,503 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,503 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,503 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,503 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN 2024-11-08T19:13:30,504 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,504 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN 2024-11-08T19:13:30,504 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-08T19:13:30,505 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=61 2024-11-08T19:13:30,506 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=692b295ad45c,36007,1731093202744, table=hbase:meta, region=1588230740 2024-11-08T19:13:30,506 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 692b295ad45c,36007,1731093202744, state=OPEN 2024-11-08T19:13:30,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,546 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,547 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,547 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,547 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,547 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN to state=OFFLINE, location=null, table=test, region=6e461a86c03ca80ab184602fce3cc56c to restore RIT 2024-11-08T19:13:30,547 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8cb2ac5239e07e076f513604c29c2ca8, ASSIGN to state=OFFLINE, location=null, table=test, region=8cb2ac5239e07e076f513604c29c2ca8 to restore RIT 2024-11-08T19:13:30,548 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731093240548 2024-11-08T19:13:30,548 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T19:13:30,548 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,549 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T19:13:30,550 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T19:13:30,550 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN 2024-11-08T19:13:30,550 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:30,550 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ea8472f395c484d4998844c4dcd304, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cb2ac5239e07e076f513604c29c2ca8, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b4a23d5efe8e832c0d5cb16df3b1369, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8e891c0073a08913834761f455a77e8, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdc84e1c2d9bdd5f39d647a57ff91a05, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,555 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53b7061bfce2939f050c57221eed7f2a, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,555 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab92690881683c86166f7f23976b5188, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,555 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d1d9fe849fd45b830c330834b58d508, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,555 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2548b6c044cd2c9ceb7dec340e2cde6c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,555 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7a8c4f9e9448f30c950ddcf996ae56f, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,555 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d0557ff953666f2f97a7be0e2d0bc87, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,555 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e461a86c03ca80ab184602fce3cc56c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,555 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=245339f649708a4ea812249dd3f822c4, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,555 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07363d44bad100b56e1b1aba358b0ed2, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,555 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0095b58ae56790b54bf50c65ac0754fa, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,555 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e92192d7abb7a6c62f8815702d7dc38d, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,556 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa4304cf54e018c1a1a2071373865b5, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,556 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1496a2d5d6037cbe8d2f7a8f1407184, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,556 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77e3781e625f0b456041758476a7d6b2, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,556 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1918a8713e3d1c571d1188f35d75a101, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,556 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=234c6774a52fc6a092eb269a4a390dc7, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,556 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7c9c940725de9fafd49c71f7bebc530, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,556 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9e50623b9f9d6b5fc998b958dd629458, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,556 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48bb25f64ce0dca8172688cf9a77f57b, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,556 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22cccf3cc4c2d3343ceae07ff16f899f, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,556 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2afe1e3e862a406c7fcb359dc1b60d57, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,556 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T19:13:30,557 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731093270556 2024-11-08T19:13:30,557 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731093330557 2024-11-08T19:13:30,557 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-08T19:13:30,557 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T19:13:30,557 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:30,557 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T19:13:30,557 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T19:13:30,557 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T19:13:30,557 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T19:13:30,557 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T19:13:30,557 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T19:13:30,558 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T19:13:30,559 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T19:13:30,560 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,560 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T19:13:30,560 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T19:13:30,560 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,560 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,560 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T19:13:30,560 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T19:13:30,560 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T19:13:30,561 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T19:13:30,562 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true 2024-11-08T19:13:30,562 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,562 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,562 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T19:13:30,562 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,36007,1731093202744 2024-11-08T19:13:30,562 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,562 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,562 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN 2024-11-08T19:13:30,562 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,562 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,562 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,562 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,563 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,36007,1731093202744 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,563 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,563 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,563 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,563 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN 2024-11-08T19:13:30,563 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-08T19:13:30,564 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=61 2024-11-08T19:13:30,564 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=692b295ad45c,36007,1731093202744, table=hbase:meta, region=1588230740 2024-11-08T19:13:30,564 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 692b295ad45c,36007,1731093202744, state=OPEN 2024-11-08T19:13:30,579 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,579 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,579 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,579 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,579 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,579 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,580 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e461a86c03ca80ab184602fce3cc56c, ASSIGN to state=OFFLINE, location=null, table=test, region=6e461a86c03ca80ab184602fce3cc56c to restore RIT 2024-11-08T19:13:30,580 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731093240580 2024-11-08T19:13:30,580 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T19:13:30,580 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,581 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T19:13:30,582 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T19:13:30,582 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,582 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:30,582 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ea8472f395c484d4998844c4dcd304, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cb2ac5239e07e076f513604c29c2ca8, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b4a23d5efe8e832c0d5cb16df3b1369, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8e891c0073a08913834761f455a77e8, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdc84e1c2d9bdd5f39d647a57ff91a05, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53b7061bfce2939f050c57221eed7f2a, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab92690881683c86166f7f23976b5188, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d1d9fe849fd45b830c330834b58d508, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2548b6c044cd2c9ceb7dec340e2cde6c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7a8c4f9e9448f30c950ddcf996ae56f, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d0557ff953666f2f97a7be0e2d0bc87, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e461a86c03ca80ab184602fce3cc56c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=245339f649708a4ea812249dd3f822c4, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07363d44bad100b56e1b1aba358b0ed2, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0095b58ae56790b54bf50c65ac0754fa, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e92192d7abb7a6c62f8815702d7dc38d, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa4304cf54e018c1a1a2071373865b5, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1496a2d5d6037cbe8d2f7a8f1407184, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77e3781e625f0b456041758476a7d6b2, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1918a8713e3d1c571d1188f35d75a101, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=234c6774a52fc6a092eb269a4a390dc7, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7c9c940725de9fafd49c71f7bebc530, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9e50623b9f9d6b5fc998b958dd629458, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48bb25f64ce0dca8172688cf9a77f57b, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22cccf3cc4c2d3343ceae07ff16f899f, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2afe1e3e862a406c7fcb359dc1b60d57, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,589 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T19:13:30,589 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731093270589 2024-11-08T19:13:30,589 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731093330589 2024-11-08T19:13:30,589 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-08T19:13:30,589 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T19:13:30,589 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:30,589 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T19:13:30,589 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T19:13:30,589 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T19:13:30,589 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T19:13:30,590 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T19:13:30,590 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T19:13:30,591 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T19:13:30,592 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T19:13:30,592 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T19:13:30,592 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T19:13:30,592 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,592 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,592 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T19:13:30,592 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T19:13:30,593 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T19:13:30,593 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T19:13:30,594 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39509 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true 2024-11-08T19:13:30,595 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,595 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,595 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T19:13:30,595 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,36007,1731093202744 2024-11-08T19:13:30,595 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,595 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,595 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,595 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,595 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,595 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,595 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,36007,1731093202744 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,595 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,595 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,595 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-08T19:13:30,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=61 2024-11-08T19:13:30,597 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=692b295ad45c,36007,1731093202744, table=hbase:meta, region=1588230740 2024-11-08T19:13:30,597 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 692b295ad45c,36007,1731093202744, state=OPEN 2024-11-08T19:13:30,601 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39509 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,608 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,609 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,609 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,609 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,610 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731093240610 2024-11-08T19:13:30,610 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T19:13:30,610 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,611 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T19:13:30,612 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T19:13:30,612 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,612 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:30,612 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,615 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ea8472f395c484d4998844c4dcd304, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,615 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cb2ac5239e07e076f513604c29c2ca8, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,615 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b4a23d5efe8e832c0d5cb16df3b1369, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,615 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8e891c0073a08913834761f455a77e8, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,615 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdc84e1c2d9bdd5f39d647a57ff91a05, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,615 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53b7061bfce2939f050c57221eed7f2a, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,616 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab92690881683c86166f7f23976b5188, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,616 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d1d9fe849fd45b830c330834b58d508, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,616 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2548b6c044cd2c9ceb7dec340e2cde6c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,616 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7a8c4f9e9448f30c950ddcf996ae56f, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,616 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d0557ff953666f2f97a7be0e2d0bc87, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,616 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e461a86c03ca80ab184602fce3cc56c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,616 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=245339f649708a4ea812249dd3f822c4, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,616 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07363d44bad100b56e1b1aba358b0ed2, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,616 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0095b58ae56790b54bf50c65ac0754fa, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,616 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e92192d7abb7a6c62f8815702d7dc38d, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa4304cf54e018c1a1a2071373865b5, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1496a2d5d6037cbe8d2f7a8f1407184, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77e3781e625f0b456041758476a7d6b2, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1918a8713e3d1c571d1188f35d75a101, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=234c6774a52fc6a092eb269a4a390dc7, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7c9c940725de9fafd49c71f7bebc530, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9e50623b9f9d6b5fc998b958dd629458, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48bb25f64ce0dca8172688cf9a77f57b, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22cccf3cc4c2d3343ceae07ff16f899f, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2afe1e3e862a406c7fcb359dc1b60d57, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,618 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T19:13:30,618 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731093270618 2024-11-08T19:13:30,618 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731093330618 2024-11-08T19:13:30,618 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-08T19:13:30,618 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T19:13:30,618 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:30,618 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T19:13:30,618 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T19:13:30,618 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T19:13:30,618 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T19:13:30,618 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T19:13:30,619 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T19:13:30,620 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T19:13:30,621 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T19:13:30,621 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T19:13:30,622 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T19:13:30,622 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,622 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,622 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T19:13:30,622 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T19:13:30,622 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T19:13:30,623 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true 2024-11-08T19:13:30,623 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,623 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,623 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T19:13:30,623 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,36007,1731093202744 2024-11-08T19:13:30,623 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,624 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,624 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,624 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,624 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,624 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,36007,1731093202744 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,624 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.1731093204811, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,624 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-08T19:13:30,625 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=61 2024-11-08T19:13:30,625 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=692b295ad45c,36007,1731093202744, table=hbase:meta, region=1588230740 2024-11-08T19:13:30,625 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 692b295ad45c,36007,1731093202744, state=OPEN 2024-11-08T19:13:30,632 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,632 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,632 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,632 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,632 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,632 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,633 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731093240633 2024-11-08T19:13:30,633 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T19:13:30,633 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,634 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T19:13:30,635 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T19:13:30,636 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,36007,1731093202744 2024-11-08T19:13:30,636 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:30,636 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,36007,1731093202744 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,638 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ea8472f395c484d4998844c4dcd304, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,638 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cb2ac5239e07e076f513604c29c2ca8, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,639 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b4a23d5efe8e832c0d5cb16df3b1369, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,639 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8e891c0073a08913834761f455a77e8, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,639 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdc84e1c2d9bdd5f39d647a57ff91a05, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,639 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53b7061bfce2939f050c57221eed7f2a, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,639 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab92690881683c86166f7f23976b5188, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,639 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d1d9fe849fd45b830c330834b58d508, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,639 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2548b6c044cd2c9ceb7dec340e2cde6c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7a8c4f9e9448f30c950ddcf996ae56f, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d0557ff953666f2f97a7be0e2d0bc87, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e461a86c03ca80ab184602fce3cc56c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=245339f649708a4ea812249dd3f822c4, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07363d44bad100b56e1b1aba358b0ed2, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0095b58ae56790b54bf50c65ac0754fa, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e92192d7abb7a6c62f8815702d7dc38d, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa4304cf54e018c1a1a2071373865b5, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1496a2d5d6037cbe8d2f7a8f1407184, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77e3781e625f0b456041758476a7d6b2, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1918a8713e3d1c571d1188f35d75a101, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=234c6774a52fc6a092eb269a4a390dc7, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7c9c940725de9fafd49c71f7bebc530, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,641 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9e50623b9f9d6b5fc998b958dd629458, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,641 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48bb25f64ce0dca8172688cf9a77f57b, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,641 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22cccf3cc4c2d3343ceae07ff16f899f, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,641 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2afe1e3e862a406c7fcb359dc1b60d57, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,641 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T19:13:30,641 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731093270641 2024-11-08T19:13:30,641 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731093330641 2024-11-08T19:13:30,641 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-08T19:13:30,641 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T19:13:30,641 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:30,641 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T19:13:30,641 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T19:13:30,642 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T19:13:30,642 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T19:13:30,642 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T19:13:30,642 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T19:13:30,643 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T19:13:30,644 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T19:13:30,644 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T19:13:30,644 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T19:13:30,644 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,644 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T19:13:30,644 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T19:13:30,645 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T19:13:30,645 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true 2024-11-08T19:13:30,646 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,646 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,646 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T19:13:30,646 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,36007,1731093202744 2024-11-08T19:13:30,646 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,646 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,646 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,646 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,646 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,36007,1731093202744 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,646 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-08T19:13:30,647 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=61 2024-11-08T19:13:30,647 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=692b295ad45c,36007,1731093202744, table=hbase:meta, region=1588230740 2024-11-08T19:13:30,647 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 692b295ad45c,36007,1731093202744, state=OPEN 2024-11-08T19:13:30,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,653 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,653 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,653 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,653 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,654 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731093240654 2024-11-08T19:13:30,654 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T19:13:30,654 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,655 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T19:13:30,656 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T19:13:30,656 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T19:13:30,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:30,656 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ea8472f395c484d4998844c4dcd304, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cb2ac5239e07e076f513604c29c2ca8, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b4a23d5efe8e832c0d5cb16df3b1369, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8e891c0073a08913834761f455a77e8, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdc84e1c2d9bdd5f39d647a57ff91a05, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53b7061bfce2939f050c57221eed7f2a, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab92690881683c86166f7f23976b5188, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d1d9fe849fd45b830c330834b58d508, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2548b6c044cd2c9ceb7dec340e2cde6c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7a8c4f9e9448f30c950ddcf996ae56f, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d0557ff953666f2f97a7be0e2d0bc87, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e461a86c03ca80ab184602fce3cc56c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=245339f649708a4ea812249dd3f822c4, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07363d44bad100b56e1b1aba358b0ed2, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0095b58ae56790b54bf50c65ac0754fa, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e92192d7abb7a6c62f8815702d7dc38d, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa4304cf54e018c1a1a2071373865b5, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1496a2d5d6037cbe8d2f7a8f1407184, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77e3781e625f0b456041758476a7d6b2, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1918a8713e3d1c571d1188f35d75a101, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=234c6774a52fc6a092eb269a4a390dc7, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7c9c940725de9fafd49c71f7bebc530, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,660 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9e50623b9f9d6b5fc998b958dd629458, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,660 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48bb25f64ce0dca8172688cf9a77f57b, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,660 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22cccf3cc4c2d3343ceae07ff16f899f, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,660 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2afe1e3e862a406c7fcb359dc1b60d57, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,660 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T19:13:30,660 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731093270660 2024-11-08T19:13:30,660 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731093330660 2024-11-08T19:13:30,660 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 6 msec 2024-11-08T19:13:30,660 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T19:13:30,660 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:30,660 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T19:13:30,660 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T19:13:30,660 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T19:13:30,660 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T19:13:30,660 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T19:13:30,661 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T19:13:30,661 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T19:13:30,662 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T19:13:30,663 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T19:13:30,663 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T19:13:30,663 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T19:13:30,663 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T19:13:30,663 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T19:13:30,663 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true 2024-11-08T19:13:30,664 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,664 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,664 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T19:13:30,664 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,664 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,664 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,664 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,664 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-08T19:13:30,664 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=61 2024-11-08T19:13:30,664 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=692b295ad45c,36007,1731093202744, table=hbase:meta, region=1588230740 2024-11-08T19:13:30,665 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 692b295ad45c,36007,1731093202744, state=OPEN 2024-11-08T19:13:30,674 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,674 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,674 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,674 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,674 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,674 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,675 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731093240675 2024-11-08T19:13:30,675 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T19:13:30,675 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,676 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T19:13:30,677 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T19:13:30,677 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,677 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:30,677 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,680 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ea8472f395c484d4998844c4dcd304, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,680 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cb2ac5239e07e076f513604c29c2ca8, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,680 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b4a23d5efe8e832c0d5cb16df3b1369, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,680 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8e891c0073a08913834761f455a77e8, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,680 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdc84e1c2d9bdd5f39d647a57ff91a05, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,680 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53b7061bfce2939f050c57221eed7f2a, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,680 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab92690881683c86166f7f23976b5188, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,680 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d1d9fe849fd45b830c330834b58d508, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,680 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2548b6c044cd2c9ceb7dec340e2cde6c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,680 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7a8c4f9e9448f30c950ddcf996ae56f, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,680 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d0557ff953666f2f97a7be0e2d0bc87, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,680 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e461a86c03ca80ab184602fce3cc56c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,680 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=245339f649708a4ea812249dd3f822c4, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,681 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07363d44bad100b56e1b1aba358b0ed2, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,681 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0095b58ae56790b54bf50c65ac0754fa, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,681 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e92192d7abb7a6c62f8815702d7dc38d, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,681 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa4304cf54e018c1a1a2071373865b5, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,681 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1496a2d5d6037cbe8d2f7a8f1407184, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,681 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77e3781e625f0b456041758476a7d6b2, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,681 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1918a8713e3d1c571d1188f35d75a101, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,681 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=234c6774a52fc6a092eb269a4a390dc7, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,681 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7c9c940725de9fafd49c71f7bebc530, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,681 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9e50623b9f9d6b5fc998b958dd629458, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,681 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48bb25f64ce0dca8172688cf9a77f57b, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,681 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22cccf3cc4c2d3343ceae07ff16f899f, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,682 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2afe1e3e862a406c7fcb359dc1b60d57, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,682 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T19:13:30,682 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731093270682 2024-11-08T19:13:30,682 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731093330682 2024-11-08T19:13:30,682 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 6 msec 2024-11-08T19:13:30,682 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T19:13:30,682 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:30,682 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T19:13:30,682 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T19:13:30,682 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T19:13:30,682 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T19:13:30,682 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T19:13:30,683 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T19:13:30,683 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T19:13:30,685 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T19:13:30,685 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T19:13:30,685 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T19:13:30,685 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T19:13:30,685 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T19:13:30,685 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true 2024-11-08T19:13:30,685 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,686 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,686 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,686 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,686 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,686 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-08T19:13:30,686 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=61 2024-11-08T19:13:30,686 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=692b295ad45c,36007,1731093202744, table=hbase:meta, region=1588230740 2024-11-08T19:13:30,687 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 692b295ad45c,36007,1731093202744, state=OPEN 2024-11-08T19:13:30,695 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,695 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,695 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,695 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,697 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731093240697 2024-11-08T19:13:30,697 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T19:13:30,697 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,698 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T19:13:30,699 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T19:13:30,699 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,699 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:30,699 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,701 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ea8472f395c484d4998844c4dcd304, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,701 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cb2ac5239e07e076f513604c29c2ca8, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,701 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b4a23d5efe8e832c0d5cb16df3b1369, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,701 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8e891c0073a08913834761f455a77e8, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,701 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdc84e1c2d9bdd5f39d647a57ff91a05, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53b7061bfce2939f050c57221eed7f2a, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab92690881683c86166f7f23976b5188, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d1d9fe849fd45b830c330834b58d508, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2548b6c044cd2c9ceb7dec340e2cde6c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7a8c4f9e9448f30c950ddcf996ae56f, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d0557ff953666f2f97a7be0e2d0bc87, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e461a86c03ca80ab184602fce3cc56c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=245339f649708a4ea812249dd3f822c4, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07363d44bad100b56e1b1aba358b0ed2, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0095b58ae56790b54bf50c65ac0754fa, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e92192d7abb7a6c62f8815702d7dc38d, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa4304cf54e018c1a1a2071373865b5, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1496a2d5d6037cbe8d2f7a8f1407184, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77e3781e625f0b456041758476a7d6b2, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1918a8713e3d1c571d1188f35d75a101, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=234c6774a52fc6a092eb269a4a390dc7, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7c9c940725de9fafd49c71f7bebc530, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9e50623b9f9d6b5fc998b958dd629458, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48bb25f64ce0dca8172688cf9a77f57b, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22cccf3cc4c2d3343ceae07ff16f899f, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2afe1e3e862a406c7fcb359dc1b60d57, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,703 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T19:13:30,703 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731093270703 2024-11-08T19:13:30,703 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731093330703 2024-11-08T19:13:30,703 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 6 msec 2024-11-08T19:13:30,703 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T19:13:30,703 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:30,703 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T19:13:30,703 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T19:13:30,704 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T19:13:30,704 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T19:13:30,704 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T19:13:30,704 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T19:13:30,705 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T19:13:30,706 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T19:13:30,706 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T19:13:30,706 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T19:13:30,706 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T19:13:30,707 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true 2024-11-08T19:13:30,707 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,707 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,707 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 692b295ad45c%2C39647%2C1731093203061.meta.1731093205333.meta, worker=692b295ad45c,42693,1731093202977 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T19:13:30,707 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-08T19:13:30,708 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=61 2024-11-08T19:13:30,708 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=692b295ad45c,36007,1731093202744, table=hbase:meta, region=1588230740 2024-11-08T19:13:30,708 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 692b295ad45c,36007,1731093202744, state=OPEN 2024-11-08T19:13:30,710 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39509 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,716 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,716 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,716 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,716 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,717 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731093240717 2024-11-08T19:13:30,717 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T19:13:30,717 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,718 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T19:13:30,719 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T19:13:30,719 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true 2024-11-08T19:13:30,719 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:30,720 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ea8472f395c484d4998844c4dcd304, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cb2ac5239e07e076f513604c29c2ca8, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b4a23d5efe8e832c0d5cb16df3b1369, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8e891c0073a08913834761f455a77e8, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdc84e1c2d9bdd5f39d647a57ff91a05, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53b7061bfce2939f050c57221eed7f2a, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab92690881683c86166f7f23976b5188, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d1d9fe849fd45b830c330834b58d508, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2548b6c044cd2c9ceb7dec340e2cde6c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7a8c4f9e9448f30c950ddcf996ae56f, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d0557ff953666f2f97a7be0e2d0bc87, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e461a86c03ca80ab184602fce3cc56c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=245339f649708a4ea812249dd3f822c4, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07363d44bad100b56e1b1aba358b0ed2, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0095b58ae56790b54bf50c65ac0754fa, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e92192d7abb7a6c62f8815702d7dc38d, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa4304cf54e018c1a1a2071373865b5, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1496a2d5d6037cbe8d2f7a8f1407184, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77e3781e625f0b456041758476a7d6b2, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1918a8713e3d1c571d1188f35d75a101, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=234c6774a52fc6a092eb269a4a390dc7, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7c9c940725de9fafd49c71f7bebc530, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,724 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9e50623b9f9d6b5fc998b958dd629458, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,724 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48bb25f64ce0dca8172688cf9a77f57b, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,724 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22cccf3cc4c2d3343ceae07ff16f899f, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,724 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2afe1e3e862a406c7fcb359dc1b60d57, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,724 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T19:13:30,724 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731093270724 2024-11-08T19:13:30,724 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731093330724 2024-11-08T19:13:30,724 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 6 msec 2024-11-08T19:13:30,724 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T19:13:30,724 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:30,724 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T19:13:30,724 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T19:13:30,724 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T19:13:30,724 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T19:13:30,724 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T19:13:30,725 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T19:13:30,726 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T19:13:30,727 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T19:13:30,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T19:13:30,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T19:13:30,728 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true 2024-11-08T19:13:30,728 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T19:13:30,728 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-08T19:13:30,728 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=61 2024-11-08T19:13:30,729 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=692b295ad45c,36007,1731093202744, table=hbase:meta, region=1588230740 2024-11-08T19:13:30,729 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 692b295ad45c,36007,1731093202744, state=OPEN 2024-11-08T19:13:30,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,737 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:13:30,737 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,737 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,738 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:13:30,739 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731093240739 2024-11-08T19:13:30,739 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T19:13:30,739 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T19:13:30,740 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T19:13:30,743 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2ea8472f395c484d4998844c4dcd304, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,743 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8cb2ac5239e07e076f513604c29c2ca8, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,743 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b4a23d5efe8e832c0d5cb16df3b1369, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,743 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a8e891c0073a08913834761f455a77e8, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,744 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdc84e1c2d9bdd5f39d647a57ff91a05, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,744 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53b7061bfce2939f050c57221eed7f2a, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,744 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab92690881683c86166f7f23976b5188, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,744 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7d1d9fe849fd45b830c330834b58d508, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,744 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2548b6c044cd2c9ceb7dec340e2cde6c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,744 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7a8c4f9e9448f30c950ddcf996ae56f, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,744 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d0557ff953666f2f97a7be0e2d0bc87, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,744 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e461a86c03ca80ab184602fce3cc56c, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,744 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=245339f649708a4ea812249dd3f822c4, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=57, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 8cb2ac5239e07e076f513604c29c2ca8, server=692b295ad45c,42693,1731093202977:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 692b295ad45c,39647,1731093203061, splitWal=true, meta=true exec-time=2.5070 sec 2024-11-08T19:13:30,745 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07363d44bad100b56e1b1aba358b0ed2, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,745 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0095b58ae56790b54bf50c65ac0754fa, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,745 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e92192d7abb7a6c62f8815702d7dc38d, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,745 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa4304cf54e018c1a1a2071373865b5, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,745 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f1496a2d5d6037cbe8d2f7a8f1407184, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,745 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77e3781e625f0b456041758476a7d6b2, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,745 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1918a8713e3d1c571d1188f35d75a101, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,745 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=234c6774a52fc6a092eb269a4a390dc7, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,745 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7c9c940725de9fafd49c71f7bebc530, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,745 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9e50623b9f9d6b5fc998b958dd629458, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,746 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48bb25f64ce0dca8172688cf9a77f57b, regionState=OPEN, lastHost=692b295ad45c,42693,1731093202977, regionLocation=692b295ad45c,42693,1731093202977, openSeqNum=2 2024-11-08T19:13:30,746 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22cccf3cc4c2d3343ceae07ff16f899f, regionState=OPENING, lastHost=692b295ad45c,39647,1731093203061, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,746 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2afe1e3e862a406c7fcb359dc1b60d57, regionState=OPEN, lastHost=692b295ad45c,36007,1731093202744, regionLocation=692b295ad45c,36007,1731093202744, openSeqNum=2 2024-11-08T19:13:30,746 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T19:13:30,746 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731093270746 2024-11-08T19:13:30,746 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731093330746 2024-11-08T19:13:30,746 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-08T19:13:30,762 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=345 (was 379), OpenFileDescriptor=622 (was 643), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=407 (was 407), ProcessCount=11 (was 11), AvailableMemoryMB=2320 (was 2370) 2024-11-08T19:13:30,762 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-08T19:13:30,762 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T19:13:30,762 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T19:13:30,762 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T19:13:30,763 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T19:13:30,763 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T19:13:30,764 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-08T19:13:30,764 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=836423166, stopped=false 2024-11-08T19:13:30,764 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=692b295ad45c,39509,1731093201567 2024-11-08T19:13:30,777 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T19:13:30,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T19:13:30,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T19:13:30,777 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:30,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T19:13:30,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T19:13:30,777 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T19:13:30,777 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T19:13:30,777 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T19:13:30,777 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T19:13:30,778 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T19:13:30,778 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T19:13:30,778 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '692b295ad45c,36007,1731093202744' ***** 2024-11-08T19:13:30,778 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-08T19:13:30,778 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T19:13:30,778 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '692b295ad45c,42693,1731093202977' ***** 2024-11-08T19:13:30,778 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-08T19:13:30,778 INFO [RS:0;692b295ad45c:36007 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T19:13:30,778 INFO [RS:1;692b295ad45c:42693 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T19:13:30,778 INFO [RS:1;692b295ad45c:42693 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T19:13:30,778 INFO [RS:0;692b295ad45c:36007 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T19:13:30,778 INFO [RS:1;692b295ad45c:42693 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T19:13:30,778 INFO [RS:0;692b295ad45c:36007 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T19:13:30,778 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T19:13:30,778 INFO [RS:1;692b295ad45c:42693 {}] regionserver.HRegionServer(3091): Received CLOSE for ab92690881683c86166f7f23976b5188 2024-11-08T19:13:30,778 INFO [RS:0;692b295ad45c:36007 {}] regionserver.HRegionServer(3091): Received CLOSE for caa4304cf54e018c1a1a2071373865b5 2024-11-08T19:13:30,779 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T19:13:30,779 INFO [RS:1;692b295ad45c:42693 {}] regionserver.HRegionServer(3091): Received CLOSE for 77e3781e625f0b456041758476a7d6b2 2024-11-08T19:13:30,779 INFO [RS:1;692b295ad45c:42693 {}] regionserver.HRegionServer(3091): Received CLOSE for e92192d7abb7a6c62f8815702d7dc38d 2024-11-08T19:13:30,779 INFO [RS:1;692b295ad45c:42693 {}] regionserver.HRegionServer(3091): Received CLOSE for a8e891c0073a08913834761f455a77e8 2024-11-08T19:13:30,779 INFO [RS:1;692b295ad45c:42693 {}] regionserver.HRegionServer(3091): Received CLOSE for 0095b58ae56790b54bf50c65ac0754fa 2024-11-08T19:13:30,779 INFO [RS:0;692b295ad45c:36007 {}] regionserver.HRegionServer(3091): Received CLOSE for 9e50623b9f9d6b5fc998b958dd629458 2024-11-08T19:13:30,779 INFO [RS:1;692b295ad45c:42693 {}] regionserver.HRegionServer(3091): Received CLOSE for 245339f649708a4ea812249dd3f822c4 2024-11-08T19:13:30,779 INFO [RS:0;692b295ad45c:36007 {}] regionserver.HRegionServer(3091): Received CLOSE for 7d1d9fe849fd45b830c330834b58d508 2024-11-08T19:13:30,779 INFO [RS:1;692b295ad45c:42693 {}] regionserver.HRegionServer(3091): Received CLOSE for f7c9c940725de9fafd49c71f7bebc530 2024-11-08T19:13:30,779 INFO [RS:0;692b295ad45c:36007 {}] regionserver.HRegionServer(3091): Received CLOSE for d2ea8472f395c484d4998844c4dcd304 2024-11-08T19:13:30,779 INFO [RS:1;692b295ad45c:42693 {}] regionserver.HRegionServer(3091): Received CLOSE for 8cb2ac5239e07e076f513604c29c2ca8 2024-11-08T19:13:30,779 INFO [RS:0;692b295ad45c:36007 {}] regionserver.HRegionServer(3091): Received CLOSE for 07363d44bad100b56e1b1aba358b0ed2 2024-11-08T19:13:30,779 INFO [RS:1;692b295ad45c:42693 {}] regionserver.HRegionServer(3091): Received CLOSE for f7a8c4f9e9448f30c950ddcf996ae56f 2024-11-08T19:13:30,779 INFO [RS:0;692b295ad45c:36007 {}] regionserver.HRegionServer(3091): Received CLOSE for 1918a8713e3d1c571d1188f35d75a101 2024-11-08T19:13:30,779 INFO [RS:1;692b295ad45c:42693 {}] regionserver.HRegionServer(3091): Received CLOSE for 48bb25f64ce0dca8172688cf9a77f57b 2024-11-08T19:13:30,779 INFO [RS:1;692b295ad45c:42693 {}] regionserver.HRegionServer(3091): Received CLOSE for 7b4a23d5efe8e832c0d5cb16df3b1369 2024-11-08T19:13:30,779 INFO [RS:0;692b295ad45c:36007 {}] regionserver.HRegionServer(3091): Received CLOSE for 2afe1e3e862a406c7fcb359dc1b60d57 2024-11-08T19:13:30,779 INFO [RS:0;692b295ad45c:36007 {}] regionserver.HRegionServer(3091): Received CLOSE for 3d0557ff953666f2f97a7be0e2d0bc87 2024-11-08T19:13:30,780 INFO [RS:0;692b295ad45c:36007 {}] regionserver.HRegionServer(3091): Received CLOSE for fdc84e1c2d9bdd5f39d647a57ff91a05 2024-11-08T19:13:30,780 INFO [RS:1;692b295ad45c:42693 {}] regionserver.HRegionServer(959): stopping server 692b295ad45c,42693,1731093202977 2024-11-08T19:13:30,780 INFO [RS:0;692b295ad45c:36007 {}] regionserver.HRegionServer(959): stopping server 692b295ad45c,36007,1731093202744 2024-11-08T19:13:30,780 INFO [RS:1;692b295ad45c:42693 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T19:13:30,780 INFO [RS:0;692b295ad45c:36007 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T19:13:30,780 INFO [RS:1;692b295ad45c:42693 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;692b295ad45c:42693. 2024-11-08T19:13:30,780 INFO [RS:0;692b295ad45c:36007 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;692b295ad45c:36007. 2024-11-08T19:13:30,780 DEBUG [RS:1;692b295ad45c:42693 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T19:13:30,780 DEBUG [RS:0;692b295ad45c:36007 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T19:13:30,780 DEBUG [RS:1;692b295ad45c:42693 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T19:13:30,780 DEBUG [RS:0;692b295ad45c:36007 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T19:13:30,780 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ab92690881683c86166f7f23976b5188, disabling compactions & flushes 2024-11-08T19:13:30,780 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing caa4304cf54e018c1a1a2071373865b5, disabling compactions & flushes 2024-11-08T19:13:30,780 INFO [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1731093205731.caa4304cf54e018c1a1a2071373865b5. 2024-11-08T19:13:30,780 INFO [RS:1;692b295ad45c:42693 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-11-08T19:13:30,780 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1731093205731.caa4304cf54e018c1a1a2071373865b5. 2024-11-08T19:13:30,780 INFO [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1731093205731.ab92690881683c86166f7f23976b5188. 2024-11-08T19:13:30,780 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1731093205731.caa4304cf54e018c1a1a2071373865b5. after waiting 0 ms 2024-11-08T19:13:30,780 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1731093205731.ab92690881683c86166f7f23976b5188. 2024-11-08T19:13:30,780 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1731093205731.caa4304cf54e018c1a1a2071373865b5. 2024-11-08T19:13:30,780 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1731093205731.ab92690881683c86166f7f23976b5188. after waiting 0 ms 2024-11-08T19:13:30,780 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1731093205731.ab92690881683c86166f7f23976b5188. 2024-11-08T19:13:30,780 INFO [RS:0;692b295ad45c:36007 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T19:13:30,780 INFO [RS:0;692b295ad45c:36007 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T19:13:30,781 INFO [RS:0;692b295ad45c:36007 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T19:13:30,781 INFO [RS:0;692b295ad45c:36007 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-08T19:13:30,781 DEBUG [RS:1;692b295ad45c:42693 {}] regionserver.HRegionServer(1351): Waiting on 0095b58ae56790b54bf50c65ac0754fa, 234c6774a52fc6a092eb269a4a390dc7, 245339f649708a4ea812249dd3f822c4, 48bb25f64ce0dca8172688cf9a77f57b, 77e3781e625f0b456041758476a7d6b2, 7b4a23d5efe8e832c0d5cb16df3b1369, 8cb2ac5239e07e076f513604c29c2ca8, a8e891c0073a08913834761f455a77e8, ab92690881683c86166f7f23976b5188, e92192d7abb7a6c62f8815702d7dc38d, f7a8c4f9e9448f30c950ddcf996ae56f, f7c9c940725de9fafd49c71f7bebc530 2024-11-08T19:13:30,781 INFO [RS:0;692b295ad45c:36007 {}] regionserver.HRegionServer(1321): Waiting on 10 regions to close 2024-11-08T19:13:30,781 DEBUG [RS:0;692b295ad45c:36007 {}] regionserver.HRegionServer(1351): Waiting on 07363d44bad100b56e1b1aba358b0ed2, 1588230740, 1918a8713e3d1c571d1188f35d75a101, 2afe1e3e862a406c7fcb359dc1b60d57, 3d0557ff953666f2f97a7be0e2d0bc87, 7d1d9fe849fd45b830c330834b58d508, 9e50623b9f9d6b5fc998b958dd629458, caa4304cf54e018c1a1a2071373865b5, d2ea8472f395c484d4998844c4dcd304, f1496a2d5d6037cbe8d2f7a8f1407184, fdc84e1c2d9bdd5f39d647a57ff91a05 2024-11-08T19:13:30,781 DEBUG [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T19:13:30,781 INFO [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T19:13:30,781 DEBUG [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T19:13:30,782 DEBUG [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T19:13:30,782 DEBUG [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T19:13:30,782 INFO [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.66 KB heapSize=5.44 KB 2024-11-08T19:13:30,786 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/ab92690881683c86166f7f23976b5188/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T19:13:30,786 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/caa4304cf54e018c1a1a2071373865b5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T19:13:30,788 INFO [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1731093205731.ab92690881683c86166f7f23976b5188. 2024-11-08T19:13:30,788 INFO [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1731093205731.caa4304cf54e018c1a1a2071373865b5. 2024-11-08T19:13:30,788 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ab92690881683c86166f7f23976b5188: Waiting for close lock at 1731093210779Running coprocessor pre-close hooks at 1731093210780 (+1 ms)Disabling compacts and flushes for region at 1731093210780Disabling writes for close at 1731093210780Writing region close event to WAL at 1731093210781 (+1 ms)Running coprocessor post-close hooks at 1731093210786 (+5 ms)Closed at 1731093210788 (+2 ms) 2024-11-08T19:13:30,788 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for caa4304cf54e018c1a1a2071373865b5: Waiting for close lock at 1731093210779Running coprocessor pre-close hooks at 1731093210780 (+1 ms)Disabling compacts and flushes for region at 1731093210780Disabling writes for close at 1731093210780Writing region close event to WAL at 1731093210781 (+1 ms)Running coprocessor post-close hooks at 1731093210787 (+6 ms)Closed at 1731093210788 (+1 ms) 2024-11-08T19:13:30,789 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1731093205731.caa4304cf54e018c1a1a2071373865b5. 2024-11-08T19:13:30,789 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1731093205731.ab92690881683c86166f7f23976b5188. 2024-11-08T19:13:30,789 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9e50623b9f9d6b5fc998b958dd629458, disabling compactions & flushes 2024-11-08T19:13:30,789 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 77e3781e625f0b456041758476a7d6b2, disabling compactions & flushes 2024-11-08T19:13:30,789 INFO [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1731093205731.9e50623b9f9d6b5fc998b958dd629458. 2024-11-08T19:13:30,789 INFO [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1731093205731.77e3781e625f0b456041758476a7d6b2. 2024-11-08T19:13:30,789 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1731093205731.9e50623b9f9d6b5fc998b958dd629458. 2024-11-08T19:13:30,789 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1731093205731.77e3781e625f0b456041758476a7d6b2. 2024-11-08T19:13:30,789 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1731093205731.77e3781e625f0b456041758476a7d6b2. after waiting 0 ms 2024-11-08T19:13:30,789 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1731093205731.9e50623b9f9d6b5fc998b958dd629458. after waiting 0 ms 2024-11-08T19:13:30,789 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1731093205731.77e3781e625f0b456041758476a7d6b2. 2024-11-08T19:13:30,789 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1731093205731.9e50623b9f9d6b5fc998b958dd629458. 2024-11-08T19:13:30,794 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/77e3781e625f0b456041758476a7d6b2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T19:13:30,795 INFO [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1731093205731.77e3781e625f0b456041758476a7d6b2. 2024-11-08T19:13:30,795 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 77e3781e625f0b456041758476a7d6b2: Waiting for close lock at 1731093210789Running coprocessor pre-close hooks at 1731093210789Disabling compacts and flushes for region at 1731093210789Disabling writes for close at 1731093210789Writing region close event to WAL at 1731093210790 (+1 ms)Running coprocessor post-close hooks at 1731093210795 (+5 ms)Closed at 1731093210795 2024-11-08T19:13:30,796 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1731093205731.77e3781e625f0b456041758476a7d6b2. 2024-11-08T19:13:30,796 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e92192d7abb7a6c62f8815702d7dc38d, disabling compactions & flushes 2024-11-08T19:13:30,796 INFO [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ppp,1731093205731.e92192d7abb7a6c62f8815702d7dc38d. 2024-11-08T19:13:30,796 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1731093205731.e92192d7abb7a6c62f8815702d7dc38d. 2024-11-08T19:13:30,796 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1731093205731.e92192d7abb7a6c62f8815702d7dc38d. after waiting 0 ms 2024-11-08T19:13:30,796 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1731093205731.e92192d7abb7a6c62f8815702d7dc38d. 2024-11-08T19:13:30,800 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/9e50623b9f9d6b5fc998b958dd629458/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T19:13:30,801 INFO [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1731093205731.9e50623b9f9d6b5fc998b958dd629458. 2024-11-08T19:13:30,801 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9e50623b9f9d6b5fc998b958dd629458: Waiting for close lock at 1731093210789Running coprocessor pre-close hooks at 1731093210789Disabling compacts and flushes for region at 1731093210789Disabling writes for close at 1731093210789Writing region close event to WAL at 1731093210791 (+2 ms)Running coprocessor post-close hooks at 1731093210801 (+10 ms)Closed at 1731093210801 2024-11-08T19:13:30,801 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1731093205731.9e50623b9f9d6b5fc998b958dd629458. 2024-11-08T19:13:30,802 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7d1d9fe849fd45b830c330834b58d508, disabling compactions & flushes 2024-11-08T19:13:30,802 INFO [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1731093205731.7d1d9fe849fd45b830c330834b58d508. 2024-11-08T19:13:30,802 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1731093205731.7d1d9fe849fd45b830c330834b58d508. 2024-11-08T19:13:30,802 DEBUG [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/hbase/meta/1588230740/.tmp/info/acd5be81afe44a0e93cecf4d24b34f40 is 138, key is test,bbb,1731093205731.8cb2ac5239e07e076f513604c29c2ca8./info:regioninfo/1731093209708/Put/seqid=0 2024-11-08T19:13:30,802 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1731093205731.7d1d9fe849fd45b830c330834b58d508. after waiting 0 ms 2024-11-08T19:13:30,802 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1731093205731.7d1d9fe849fd45b830c330834b58d508. 2024-11-08T19:13:30,811 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/e92192d7abb7a6c62f8815702d7dc38d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T19:13:30,812 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/7d1d9fe849fd45b830c330834b58d508/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T19:13:30,812 INFO [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ppp,1731093205731.e92192d7abb7a6c62f8815702d7dc38d. 2024-11-08T19:13:30,812 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e92192d7abb7a6c62f8815702d7dc38d: Waiting for close lock at 1731093210796Running coprocessor pre-close hooks at 1731093210796Disabling compacts and flushes for region at 1731093210796Disabling writes for close at 1731093210796Writing region close event to WAL at 1731093210800 (+4 ms)Running coprocessor post-close hooks at 1731093210812 (+12 ms)Closed at 1731093210812 2024-11-08T19:13:30,813 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ppp,1731093205731.e92192d7abb7a6c62f8815702d7dc38d. 2024-11-08T19:13:30,813 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a8e891c0073a08913834761f455a77e8, disabling compactions & flushes 2024-11-08T19:13:30,813 INFO [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1731093205731.7d1d9fe849fd45b830c330834b58d508. 2024-11-08T19:13:30,813 INFO [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1731093205731.a8e891c0073a08913834761f455a77e8. 2024-11-08T19:13:30,813 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1731093205731.a8e891c0073a08913834761f455a77e8. 2024-11-08T19:13:30,813 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7d1d9fe849fd45b830c330834b58d508: Waiting for close lock at 1731093210802Running coprocessor pre-close hooks at 1731093210802Disabling compacts and flushes for region at 1731093210802Disabling writes for close at 1731093210802Writing region close event to WAL at 1731093210804 (+2 ms)Running coprocessor post-close hooks at 1731093210813 (+9 ms)Closed at 1731093210813 2024-11-08T19:13:30,813 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1731093205731.a8e891c0073a08913834761f455a77e8. after waiting 0 ms 2024-11-08T19:13:30,813 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1731093205731.a8e891c0073a08913834761f455a77e8. 2024-11-08T19:13:30,813 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1731093205731.7d1d9fe849fd45b830c330834b58d508. 2024-11-08T19:13:30,813 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d2ea8472f395c484d4998844c4dcd304, disabling compactions & flushes 2024-11-08T19:13:30,813 INFO [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1731093205731.d2ea8472f395c484d4998844c4dcd304. 2024-11-08T19:13:30,813 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1731093205731.d2ea8472f395c484d4998844c4dcd304. 2024-11-08T19:13:30,813 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1731093205731.d2ea8472f395c484d4998844c4dcd304. after waiting 0 ms 2024-11-08T19:13:30,813 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1731093205731.d2ea8472f395c484d4998844c4dcd304. 2024-11-08T19:13:30,825 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/d2ea8472f395c484d4998844c4dcd304/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T19:13:30,825 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/a8e891c0073a08913834761f455a77e8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T19:13:30,827 INFO [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1731093205731.a8e891c0073a08913834761f455a77e8. 2024-11-08T19:13:30,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741869_1045 (size=8195) 2024-11-08T19:13:30,827 INFO [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1731093205731.d2ea8472f395c484d4998844c4dcd304. 2024-11-08T19:13:30,827 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a8e891c0073a08913834761f455a77e8: Waiting for close lock at 1731093210813Running coprocessor pre-close hooks at 1731093210813Disabling compacts and flushes for region at 1731093210813Disabling writes for close at 1731093210813Writing region close event to WAL at 1731093210816 (+3 ms)Running coprocessor post-close hooks at 1731093210826 (+10 ms)Closed at 1731093210827 (+1 ms) 2024-11-08T19:13:30,827 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d2ea8472f395c484d4998844c4dcd304: Waiting for close lock at 1731093210813Running coprocessor pre-close hooks at 1731093210813Disabling compacts and flushes for region at 1731093210813Disabling writes for close at 1731093210813Writing region close event to WAL at 1731093210816 (+3 ms)Running coprocessor post-close hooks at 1731093210827 (+11 ms)Closed at 1731093210827 2024-11-08T19:13:30,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741869_1045 (size=8195) 2024-11-08T19:13:30,827 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1731093205731.d2ea8472f395c484d4998844c4dcd304. 2024-11-08T19:13:30,827 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1731093205731.a8e891c0073a08913834761f455a77e8. 2024-11-08T19:13:30,827 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0095b58ae56790b54bf50c65ac0754fa, disabling compactions & flushes 2024-11-08T19:13:30,827 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 07363d44bad100b56e1b1aba358b0ed2, disabling compactions & flushes 2024-11-08T19:13:30,827 INFO [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1731093205731.0095b58ae56790b54bf50c65ac0754fa. 2024-11-08T19:13:30,827 INFO [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1731093205731.07363d44bad100b56e1b1aba358b0ed2. 2024-11-08T19:13:30,827 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1731093205731.0095b58ae56790b54bf50c65ac0754fa. 2024-11-08T19:13:30,827 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1731093205731.07363d44bad100b56e1b1aba358b0ed2. 2024-11-08T19:13:30,827 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1731093205731.0095b58ae56790b54bf50c65ac0754fa. after waiting 0 ms 2024-11-08T19:13:30,827 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1731093205731.07363d44bad100b56e1b1aba358b0ed2. after waiting 0 ms 2024-11-08T19:13:30,827 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1731093205731.0095b58ae56790b54bf50c65ac0754fa. 2024-11-08T19:13:30,827 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1731093205731.07363d44bad100b56e1b1aba358b0ed2. 2024-11-08T19:13:30,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741869_1045 (size=8195) 2024-11-08T19:13:30,829 INFO [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.66 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/hbase/meta/1588230740/.tmp/info/acd5be81afe44a0e93cecf4d24b34f40 2024-11-08T19:13:30,833 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/07363d44bad100b56e1b1aba358b0ed2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T19:13:30,836 INFO [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1731093205731.07363d44bad100b56e1b1aba358b0ed2. 2024-11-08T19:13:30,836 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 07363d44bad100b56e1b1aba358b0ed2: Waiting for close lock at 1731093210827Running coprocessor pre-close hooks at 1731093210827Disabling compacts and flushes for region at 1731093210827Disabling writes for close at 1731093210827Writing region close event to WAL at 1731093210828 (+1 ms)Running coprocessor post-close hooks at 1731093210836 (+8 ms)Closed at 1731093210836 2024-11-08T19:13:30,836 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1731093205731.07363d44bad100b56e1b1aba358b0ed2. 2024-11-08T19:13:30,837 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1918a8713e3d1c571d1188f35d75a101, disabling compactions & flushes 2024-11-08T19:13:30,837 INFO [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1731093205731.1918a8713e3d1c571d1188f35d75a101. 2024-11-08T19:13:30,837 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1731093205731.1918a8713e3d1c571d1188f35d75a101. 2024-11-08T19:13:30,837 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1731093205731.1918a8713e3d1c571d1188f35d75a101. after waiting 0 ms 2024-11-08T19:13:30,837 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1731093205731.1918a8713e3d1c571d1188f35d75a101. 2024-11-08T19:13:30,837 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/0095b58ae56790b54bf50c65ac0754fa/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T19:13:30,838 INFO [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1731093205731.0095b58ae56790b54bf50c65ac0754fa. 2024-11-08T19:13:30,838 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0095b58ae56790b54bf50c65ac0754fa: Waiting for close lock at 1731093210827Running coprocessor pre-close hooks at 1731093210827Disabling compacts and flushes for region at 1731093210827Disabling writes for close at 1731093210827Writing region close event to WAL at 1731093210829 (+2 ms)Running coprocessor post-close hooks at 1731093210838 (+9 ms)Closed at 1731093210838 2024-11-08T19:13:30,838 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1731093205731.0095b58ae56790b54bf50c65ac0754fa. 2024-11-08T19:13:30,838 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 245339f649708a4ea812249dd3f822c4, disabling compactions & flushes 2024-11-08T19:13:30,838 INFO [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1731093205731.245339f649708a4ea812249dd3f822c4. 2024-11-08T19:13:30,839 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1731093205731.245339f649708a4ea812249dd3f822c4. 2024-11-08T19:13:30,839 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1731093205731.245339f649708a4ea812249dd3f822c4. after waiting 0 ms 2024-11-08T19:13:30,839 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1731093205731.245339f649708a4ea812249dd3f822c4. 2024-11-08T19:13:30,843 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/245339f649708a4ea812249dd3f822c4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T19:13:30,843 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/1918a8713e3d1c571d1188f35d75a101/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T19:13:30,844 DEBUG [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/hbase/meta/1588230740/.tmp/info/acd5be81afe44a0e93cecf4d24b34f40 as hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/hbase/meta/1588230740/info/acd5be81afe44a0e93cecf4d24b34f40 2024-11-08T19:13:30,844 INFO [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1731093205731.245339f649708a4ea812249dd3f822c4. 2024-11-08T19:13:30,844 INFO [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1731093205731.1918a8713e3d1c571d1188f35d75a101. 2024-11-08T19:13:30,844 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 245339f649708a4ea812249dd3f822c4: Waiting for close lock at 1731093210838Running coprocessor pre-close hooks at 1731093210838Disabling compacts and flushes for region at 1731093210838Disabling writes for close at 1731093210839 (+1 ms)Writing region close event to WAL at 1731093210839Running coprocessor post-close hooks at 1731093210844 (+5 ms)Closed at 1731093210844 2024-11-08T19:13:30,844 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1918a8713e3d1c571d1188f35d75a101: Waiting for close lock at 1731093210837Running coprocessor pre-close hooks at 1731093210837Disabling compacts and flushes for region at 1731093210837Disabling writes for close at 1731093210837Writing region close event to WAL at 1731093210838 (+1 ms)Running coprocessor post-close hooks at 1731093210844 (+6 ms)Closed at 1731093210844 2024-11-08T19:13:30,844 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1731093205731.245339f649708a4ea812249dd3f822c4. 2024-11-08T19:13:30,844 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1731093205731.1918a8713e3d1c571d1188f35d75a101. 2024-11-08T19:13:30,844 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2afe1e3e862a406c7fcb359dc1b60d57, disabling compactions & flushes 2024-11-08T19:13:30,844 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f7c9c940725de9fafd49c71f7bebc530, disabling compactions & flushes 2024-11-08T19:13:30,845 INFO [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1731093205731.2afe1e3e862a406c7fcb359dc1b60d57. 2024-11-08T19:13:30,845 INFO [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1731093205731.f7c9c940725de9fafd49c71f7bebc530. 2024-11-08T19:13:30,845 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1731093205731.2afe1e3e862a406c7fcb359dc1b60d57. 2024-11-08T19:13:30,845 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1731093205731.f7c9c940725de9fafd49c71f7bebc530. 2024-11-08T19:13:30,845 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1731093205731.f7c9c940725de9fafd49c71f7bebc530. after waiting 0 ms 2024-11-08T19:13:30,845 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1731093205731.2afe1e3e862a406c7fcb359dc1b60d57. after waiting 0 ms 2024-11-08T19:13:30,845 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1731093205731.f7c9c940725de9fafd49c71f7bebc530. 2024-11-08T19:13:30,845 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1731093205731.2afe1e3e862a406c7fcb359dc1b60d57. 2024-11-08T19:13:30,851 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/f7c9c940725de9fafd49c71f7bebc530/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T19:13:30,852 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/2afe1e3e862a406c7fcb359dc1b60d57/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T19:13:30,852 INFO [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1731093205731.f7c9c940725de9fafd49c71f7bebc530. 2024-11-08T19:13:30,852 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f7c9c940725de9fafd49c71f7bebc530: Waiting for close lock at 1731093210844Running coprocessor pre-close hooks at 1731093210844Disabling compacts and flushes for region at 1731093210844Disabling writes for close at 1731093210845 (+1 ms)Writing region close event to WAL at 1731093210845Running coprocessor post-close hooks at 1731093210852 (+7 ms)Closed at 1731093210852 2024-11-08T19:13:30,852 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1731093205731.f7c9c940725de9fafd49c71f7bebc530. 2024-11-08T19:13:30,852 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8cb2ac5239e07e076f513604c29c2ca8, disabling compactions & flushes 2024-11-08T19:13:30,852 INFO [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1731093205731.8cb2ac5239e07e076f513604c29c2ca8. 2024-11-08T19:13:30,852 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1731093205731.8cb2ac5239e07e076f513604c29c2ca8. 2024-11-08T19:13:30,852 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1731093205731.8cb2ac5239e07e076f513604c29c2ca8. after waiting 0 ms 2024-11-08T19:13:30,852 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1731093205731.8cb2ac5239e07e076f513604c29c2ca8. 2024-11-08T19:13:30,853 INFO [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1731093205731.2afe1e3e862a406c7fcb359dc1b60d57. 2024-11-08T19:13:30,853 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2afe1e3e862a406c7fcb359dc1b60d57: Waiting for close lock at 1731093210844Running coprocessor pre-close hooks at 1731093210844Disabling compacts and flushes for region at 1731093210844Disabling writes for close at 1731093210845 (+1 ms)Writing region close event to WAL at 1731093210847 (+2 ms)Running coprocessor post-close hooks at 1731093210853 (+6 ms)Closed at 1731093210853 2024-11-08T19:13:30,853 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1731093205731.2afe1e3e862a406c7fcb359dc1b60d57. 2024-11-08T19:13:30,853 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3d0557ff953666f2f97a7be0e2d0bc87, disabling compactions & flushes 2024-11-08T19:13:30,853 INFO [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1731093205731.3d0557ff953666f2f97a7be0e2d0bc87. 2024-11-08T19:13:30,853 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1731093205731.3d0557ff953666f2f97a7be0e2d0bc87. 2024-11-08T19:13:30,853 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1731093205731.3d0557ff953666f2f97a7be0e2d0bc87. after waiting 0 ms 2024-11-08T19:13:30,853 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1731093205731.3d0557ff953666f2f97a7be0e2d0bc87. 2024-11-08T19:13:30,855 INFO [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/hbase/meta/1588230740/info/acd5be81afe44a0e93cecf4d24b34f40, entries=24, sequenceid=71, filesize=8.0 K 2024-11-08T19:13:30,856 INFO [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 74ms, sequenceid=71, compaction requested=false 2024-11-08T19:13:30,860 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/3d0557ff953666f2f97a7be0e2d0bc87/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T19:13:30,860 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/8cb2ac5239e07e076f513604c29c2ca8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T19:13:30,860 INFO [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1731093205731.3d0557ff953666f2f97a7be0e2d0bc87. 2024-11-08T19:13:30,860 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3d0557ff953666f2f97a7be0e2d0bc87: Waiting for close lock at 1731093210853Running coprocessor pre-close hooks at 1731093210853Disabling compacts and flushes for region at 1731093210853Disabling writes for close at 1731093210853Writing region close event to WAL at 1731093210854 (+1 ms)Running coprocessor post-close hooks at 1731093210860 (+6 ms)Closed at 1731093210860 2024-11-08T19:13:30,861 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1731093205731.3d0557ff953666f2f97a7be0e2d0bc87. 2024-11-08T19:13:30,861 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fdc84e1c2d9bdd5f39d647a57ff91a05, disabling compactions & flushes 2024-11-08T19:13:30,861 INFO [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1731093205731.fdc84e1c2d9bdd5f39d647a57ff91a05. 2024-11-08T19:13:30,861 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1731093205731.fdc84e1c2d9bdd5f39d647a57ff91a05. 2024-11-08T19:13:30,861 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1731093205731.fdc84e1c2d9bdd5f39d647a57ff91a05. after waiting 0 ms 2024-11-08T19:13:30,861 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1731093205731.fdc84e1c2d9bdd5f39d647a57ff91a05. 2024-11-08T19:13:30,861 INFO [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1731093205731.8cb2ac5239e07e076f513604c29c2ca8. 2024-11-08T19:13:30,861 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8cb2ac5239e07e076f513604c29c2ca8: Waiting for close lock at 1731093210852Running coprocessor pre-close hooks at 1731093210852Disabling compacts and flushes for region at 1731093210852Disabling writes for close at 1731093210852Writing region close event to WAL at 1731093210853 (+1 ms)Running coprocessor post-close hooks at 1731093210861 (+8 ms)Closed at 1731093210861 2024-11-08T19:13:30,861 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1731093205731.8cb2ac5239e07e076f513604c29c2ca8. 2024-11-08T19:13:30,861 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f7a8c4f9e9448f30c950ddcf996ae56f, disabling compactions & flushes 2024-11-08T19:13:30,861 INFO [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1731093205731.f7a8c4f9e9448f30c950ddcf996ae56f. 2024-11-08T19:13:30,861 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1731093205731.f7a8c4f9e9448f30c950ddcf996ae56f. 2024-11-08T19:13:30,861 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1731093205731.f7a8c4f9e9448f30c950ddcf996ae56f. after waiting 0 ms 2024-11-08T19:13:30,861 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1731093205731.f7a8c4f9e9448f30c950ddcf996ae56f. 2024-11-08T19:13:30,863 INFO [regionserver/692b295ad45c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T19:13:30,864 DEBUG [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/hbase/meta/1588230740/recovered.edits/74.seqid, newMaxSeqId=74, maxSeqId=60 2024-11-08T19:13:30,865 INFO [regionserver/692b295ad45c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T19:13:30,865 DEBUG [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T19:13:30,865 INFO [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T19:13:30,865 DEBUG [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731093210781Running coprocessor pre-close hooks at 1731093210781Disabling compacts and flushes for region at 1731093210781Disabling writes for close at 1731093210782 (+1 ms)Obtaining lock to block concurrent updates at 1731093210782Preparing flush snapshotting stores in 1588230740 at 1731093210782Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2720, getHeapSize=5504, getOffHeapSize=0, getCellsCount=24 at 1731093210790 (+8 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731093210791 (+1 ms)Flushing 1588230740/info: creating writer at 1731093210791Flushing 1588230740/info: appending metadata at 1731093210801 (+10 ms)Flushing 1588230740/info: closing flushed file at 1731093210801Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7830edca: reopening flushed file at 1731093210839 (+38 ms)Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 74ms, sequenceid=71, compaction requested=false at 1731093210857 (+18 ms)Writing region close event to WAL at 1731093210858 (+1 ms)Running coprocessor post-close hooks at 1731093210865 (+7 ms)Closed at 1731093210865 2024-11-08T19:13:30,865 DEBUG [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-08T19:13:30,866 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/fdc84e1c2d9bdd5f39d647a57ff91a05/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T19:13:30,866 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/f7a8c4f9e9448f30c950ddcf996ae56f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T19:13:30,866 INFO [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1731093205731.fdc84e1c2d9bdd5f39d647a57ff91a05. 2024-11-08T19:13:30,867 INFO [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1731093205731.f7a8c4f9e9448f30c950ddcf996ae56f. 2024-11-08T19:13:30,867 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fdc84e1c2d9bdd5f39d647a57ff91a05: Waiting for close lock at 1731093210861Running coprocessor pre-close hooks at 1731093210861Disabling compacts and flushes for region at 1731093210861Disabling writes for close at 1731093210861Writing region close event to WAL at 1731093210861Running coprocessor post-close hooks at 1731093210866 (+5 ms)Closed at 1731093210866 2024-11-08T19:13:30,867 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f7a8c4f9e9448f30c950ddcf996ae56f: Waiting for close lock at 1731093210861Running coprocessor pre-close hooks at 1731093210861Disabling compacts and flushes for region at 1731093210861Disabling writes for close at 1731093210861Writing region close event to WAL at 1731093210862 (+1 ms)Running coprocessor post-close hooks at 1731093210866 (+4 ms)Closed at 1731093210866 2024-11-08T19:13:30,867 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1731093205731.f7a8c4f9e9448f30c950ddcf996ae56f. 2024-11-08T19:13:30,867 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1731093205731.fdc84e1c2d9bdd5f39d647a57ff91a05. 2024-11-08T19:13:30,867 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 48bb25f64ce0dca8172688cf9a77f57b, disabling compactions & flushes 2024-11-08T19:13:30,867 INFO [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1731093205731.48bb25f64ce0dca8172688cf9a77f57b. 2024-11-08T19:13:30,867 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1731093205731.48bb25f64ce0dca8172688cf9a77f57b. 2024-11-08T19:13:30,867 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1731093205731.48bb25f64ce0dca8172688cf9a77f57b. after waiting 0 ms 2024-11-08T19:13:30,867 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1731093205731.48bb25f64ce0dca8172688cf9a77f57b. 2024-11-08T19:13:30,871 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/48bb25f64ce0dca8172688cf9a77f57b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T19:13:30,871 INFO [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1731093205731.48bb25f64ce0dca8172688cf9a77f57b. 2024-11-08T19:13:30,871 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 48bb25f64ce0dca8172688cf9a77f57b: Waiting for close lock at 1731093210867Running coprocessor pre-close hooks at 1731093210867Disabling compacts and flushes for region at 1731093210867Disabling writes for close at 1731093210867Writing region close event to WAL at 1731093210867Running coprocessor post-close hooks at 1731093210871 (+4 ms)Closed at 1731093210871 2024-11-08T19:13:30,872 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1731093205731.48bb25f64ce0dca8172688cf9a77f57b. 2024-11-08T19:13:30,872 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7b4a23d5efe8e832c0d5cb16df3b1369, disabling compactions & flushes 2024-11-08T19:13:30,872 INFO [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1731093205731.7b4a23d5efe8e832c0d5cb16df3b1369. 2024-11-08T19:13:30,872 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1731093205731.7b4a23d5efe8e832c0d5cb16df3b1369. 2024-11-08T19:13:30,872 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1731093205731.7b4a23d5efe8e832c0d5cb16df3b1369. after waiting 0 ms 2024-11-08T19:13:30,872 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1731093205731.7b4a23d5efe8e832c0d5cb16df3b1369. 2024-11-08T19:13:30,876 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/7b4a23d5efe8e832c0d5cb16df3b1369/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T19:13:30,877 INFO [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1731093205731.7b4a23d5efe8e832c0d5cb16df3b1369. 2024-11-08T19:13:30,877 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7b4a23d5efe8e832c0d5cb16df3b1369: Waiting for close lock at 1731093210872Running coprocessor pre-close hooks at 1731093210872Disabling compacts and flushes for region at 1731093210872Disabling writes for close at 1731093210872Writing region close event to WAL at 1731093210872Running coprocessor post-close hooks at 1731093210877 (+5 ms)Closed at 1731093210877 2024-11-08T19:13:30,877 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1731093205731.7b4a23d5efe8e832c0d5cb16df3b1369. 2024-11-08T19:13:30,880 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T19:13:30,941 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-11-08T19:13:30,944 WARN [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,uuu,1731093205731.234c6774a52fc6a092eb269a4a390dc7., aborting... java.io.IOException: Failed to report opened region to master: test,uuu,1731093205731.234c6774a52fc6a092eb269a4a390dc7. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T19:13:30,944 ERROR [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2440): ***** ABORTING region server 692b295ad45c,42693,1731093202977: Failed to open region test,uuu,1731093205731.234c6774a52fc6a092eb269a4a390dc7. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,uuu,1731093205731.234c6774a52fc6a092eb269a4a390dc7. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T19:13:30,944 ERROR [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-08T19:13:30,945 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-08T19:13:30,946 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-08T19:13:30,948 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-08T19:13:30,950 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-08T19:13:30,958 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 468187456 }, "NonHeapMemoryUsage": { "committed": 162463744, "init": 7667712, "max": -1, "used": 159932528 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "692b295ad45c", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 40669, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 132, "ProcessCallTime_min": 0, "ProcessCallTime_max": 29, "ProcessCallTime_mean": 2, "ProcessCallTime_25th_percentile": 7, "ProcessCallTime_median": 14, "ProcessCallTime_75th_percentile": 21, "ProcessCallTime_90th_percentile": 26, "ProcessCallTime_95th_percentile": 27, "ProcessCallTime_98th_percentile": 28, "ProcessCallTime_99th_percentile": 28, "ProcessCallTime_99.9th_percentile": 28, "ProcessCallTime_TimeRangeCount_0-1": 132, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 132, "QueueCallTime_min": 0, "QueueCallTime_max": 16, "QueueCallTime_mean": 1, "QueueCallTime_25th_percentile": 4, "QueueCallTime_median": 8, "QueueCallTime_75th_percentile": 12, "QueueCallTime_90th_percentile": 14, "QueueCallTime_95th_percentile": 15, "QueueCallTime_98th_percentile": 15, "QueueCallTime_99th_percentile": 15, "QueueCallTime_99.9th_percentile": 15, "QueueCallTime_TimeRangeCount_0-1": 132, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 132, "TotalCallTime_min": 0, "TotalCallTime_max": 29, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 7, "TotalCallTime_median": 14, "TotalCallTime_75th_percentile": 21, "TotalCallTime_90th_percentile": 26, "TotalCallTime_95th_percentile": 27, "TotalCallTime_98th_percentile": 28, "TotalCallTime_99th_percentile": 28, "TotalCallTime_99.9th_percentile": 28, "TotalCallTime_TimeRangeCount_0-1": 132, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 132, "ResponseSize_min": 0, "ResponseSize_max": 22854, "ResponseSize_mean": 4500, "ResponseSize_25th_percentile": 5713, "ResponseSize_median": 11427, "ResponseSize_75th_percentile": 17140, "ResponseSize_90th_percentile": 20568, "ResponseSize_95th_percentile": 21711, "ResponseSize_98th_percentile": 22396, "ResponseSize_99th_percentile": 22625, "ResponseSize_99.9th_percentile": 22831, "ResponseSize_SizeRangeCount_0-10": 132, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 132, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 279, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 132, "sentBytes": 459895 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "692b295ad45c", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:56395", "tag.serverName": "692b295ad45c,39647,1731093203061", "tag.clusterId": "c2739604-5628-4fa2-a3e8-33881b328b44", "tag.Context": "regionserver", "tag.Hostname": "692b295ad45c", "regionCount": 9, "storeCount": 12, "hlogFileCount": 2, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 28672, "memStoreHeapSize": 51520, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731093203061, "averageRegionSize": 3185, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 38203, "localBytesRead": 38203, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 18.8, "writeRequestRatePerSecond": 16.400000000000002, "ByteBuffAllocatorHeapAllocationBytes": 39391, "ByteBuffAllocatorPoolAllocationBytes": 465920, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 74, "totalRowActionRequestCount": 176, "readRequestCount": 94, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 82, "rpcGetRequestCount": 2, "rpcFullScanRequestCount": 8, "rpcScanRequestCount": 15, "rpcMultiRequestCount": 1, "rpcMutateRequestCount": 56, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 15, "ScanTime_min": 0, "ScanTime_max": 12, "ScanTime_mean": 1, "ScanTime_25th_percentile": 3, "ScanTime_median": 6, "ScanTime_75th_percentile": 9, "ScanTime_90th_percentile": 10, "ScanTime_95th_percentile": 11, "ScanTime_98th_percentile": 11, "ScanTime_99th_percentile": 11, "ScanTime_99.9th_percentile": 11, "ScanTime_TimeRangeCount_0-1": 15, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 56, "Put_min": 1, "Put_max": 8, "Put_mean": 2, "Put_25th_percentile": 2, "Put_median": 4, "Put_75th_percentile": 6, "Put_90th_percentile": 7, "Put_95th_percentile": 7, "Put_98th_percentile": 7, "Put_99th_percentile": 7, "Put_99.9th_percentile": 7, "Put_TimeRangeCount_0-1": 56, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 1, "PutBatch_min": 11, "PutBatch_max": 11, "PutBatch_mean": 11, "PutBatch_25th_percentile": 11, "PutBatch_median": 11, "PutBatch_75th_percentile": 11, "PutBatch_90th_percentile": 11, "PutBatch_95th_percentile": 11, "PutBatch_98th_percentile": 11, "PutBatch_99th_percentile": 11, "PutBatch_99.9th_percentile": 11, "PutBatch_TimeRangeCount_0-1": 1, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 2, "Get_min": 1, "Get_max": 2, "Get_mean": 1, "Get_25th_percentile": 1, "Get_median": 1, "Get_75th_percentile": 1, "Get_90th_percentile": 1, "Get_95th_percentile": 1, "Get_98th_percentile": 1, "Get_99th_percentile": 1, "Get_99.9th_percentile": 1, "Get_TimeRangeCount_0-1": 2, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 15, "ScanSize_min": 0, "ScanSize_max": 22232, "ScanSize_mean": 4465, "ScanSize_25th_percentile": 5558, "ScanSize_median": 11116, "ScanSize_75th_percentile": 16674, "ScanSize_90th_percentile": 20008, "ScanSize_95th_percentile": 21120, "ScanSize_98th_percentile": 21787, "ScanSize_99th_percentile": 22009, "ScanSize_99.9th_percentile": 22209, "ScanSize_SizeRangeCount_0-10": 15, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-08T19:13:30,960 WARN [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2470): Unable to report fatal error to master org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=692b295ad45c:39509 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:344) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportRSFatalError(RegionServerStatusProtos.java:17290) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2467) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abortRegionServer(SingleProcessHBaseCluster.java:192) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$2.run(SingleProcessHBaseCluster.java:185) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abort(SingleProcessHBaseCluster.java:182) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.handleException(AssignRegionHandler.java:173) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:106) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=692b295ad45c:39509 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 15 more Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 15 more 2024-11-08T19:13:30,961 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(132): Open test,iii,1731093205731.2548b6c044cd2c9ceb7dec340e2cde6c. 2024-11-08T19:13:30,962 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7752): Opening region: {ENCODED => 2548b6c044cd2c9ceb7dec340e2cde6c, NAME => 'test,iii,1731093205731.2548b6c044cd2c9ceb7dec340e2cde6c.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-08T19:13:30,962 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2548b6c044cd2c9ceb7dec340e2cde6c 2024-11-08T19:13:30,962 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(898): Instantiated test,iii,1731093205731.2548b6c044cd2c9ceb7dec340e2cde6c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:13:30,963 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7794): checking encryption for 2548b6c044cd2c9ceb7dec340e2cde6c 2024-11-08T19:13:30,963 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7797): checking classloading for 2548b6c044cd2c9ceb7dec340e2cde6c 2024-11-08T19:13:30,965 INFO [StoreOpener-2548b6c044cd2c9ceb7dec340e2cde6c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2548b6c044cd2c9ceb7dec340e2cde6c 2024-11-08T19:13:30,966 INFO [StoreOpener-2548b6c044cd2c9ceb7dec340e2cde6c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2548b6c044cd2c9ceb7dec340e2cde6c columnFamilyName family 2024-11-08T19:13:30,966 DEBUG [StoreOpener-2548b6c044cd2c9ceb7dec340e2cde6c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:13:30,967 INFO [StoreOpener-2548b6c044cd2c9ceb7dec340e2cde6c-1 {}] regionserver.HStore(327): Store=2548b6c044cd2c9ceb7dec340e2cde6c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T19:13:30,967 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1038): replaying wal for 2548b6c044cd2c9ceb7dec340e2cde6c 2024-11-08T19:13:30,968 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/2548b6c044cd2c9ceb7dec340e2cde6c 2024-11-08T19:13:30,969 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/data/default/test/2548b6c044cd2c9ceb7dec340e2cde6c 2024-11-08T19:13:30,970 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1048): stopping wal replay for 2548b6c044cd2c9ceb7dec340e2cde6c 2024-11-08T19:13:30,970 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1060): Cleaning up temporary data for 2548b6c044cd2c9ceb7dec340e2cde6c 2024-11-08T19:13:30,972 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1093): writing seq id for 2548b6c044cd2c9ceb7dec340e2cde6c 2024-11-08T19:13:30,973 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1114): Opened 2548b6c044cd2c9ceb7dec340e2cde6c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67070658, jitterRate=-5.693137645721436E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T19:13:30,973 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2548b6c044cd2c9ceb7dec340e2cde6c 2024-11-08T19:13:30,973 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1006): Region open journal for 2548b6c044cd2c9ceb7dec340e2cde6c: Running coprocessor pre-open hook at 1731093210963Writing region info on filesystem at 1731093210963Initializing all the Stores at 1731093210964 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731093210964Cleaning up temporary data from old regions at 1731093210970 (+6 ms)Running coprocessor post-open hooks at 1731093210973 (+3 ms)Region opened successfully at 1731093210973 2024-11-08T19:13:30,975 WARN [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,iii,1731093205731.2548b6c044cd2c9ceb7dec340e2cde6c., aborting... org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 692b295ad45c,42693,1731093202977 aborting at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkOpen(RSRpcServices.java:1463) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2235) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T19:13:30,975 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2435): Abort already in progress. Ignoring the current request with reason: Failed to open region test,iii,1731093205731.2548b6c044cd2c9ceb7dec340e2cde6c. and can not recover 2024-11-08T19:13:30,981 INFO [RS:1;692b295ad45c:42693 {}] regionserver.HRegionServer(976): stopping server 692b295ad45c,42693,1731093202977; all regions closed. 2024-11-08T19:13:30,981 INFO [RS:0;692b295ad45c:36007 {}] regionserver.HRegionServer(976): stopping server 692b295ad45c,36007,1731093202744; all regions closed. 2024-11-08T19:13:30,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741833_1009 (size=6536) 2024-11-08T19:13:30,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741833_1009 (size=6536) 2024-11-08T19:13:30,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741833_1009 (size=6536) 2024-11-08T19:13:30,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741865_1041 (size=4204) 2024-11-08T19:13:30,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741865_1041 (size=4204) 2024-11-08T19:13:30,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741865_1041 (size=4204) 2024-11-08T19:13:30,988 DEBUG [RS:0;692b295ad45c:36007 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/oldWALs 2024-11-08T19:13:30,989 INFO [RS:0;692b295ad45c:36007 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 692b295ad45c%2C36007%2C1731093202744.meta:.meta(num 1731093208921) 2024-11-08T19:13:30,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741835_1011 (size=5184) 2024-11-08T19:13:30,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741835_1011 (size=5184) 2024-11-08T19:13:30,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741835_1011 (size=5184) 2024-11-08T19:13:30,992 DEBUG [RS:1;692b295ad45c:42693 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T19:13:30,992 INFO [RS:1;692b295ad45c:42693 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T19:13:30,992 INFO [RS:1;692b295ad45c:42693 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T19:13:30,993 INFO [RS:1;692b295ad45c:42693 {}] hbase.ChoreService(370): Chore service for: regionserver/692b295ad45c:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-08T19:13:30,993 INFO [RS:1;692b295ad45c:42693 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T19:13:30,993 INFO [RS:1;692b295ad45c:42693 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T19:13:30,993 INFO [RS:1;692b295ad45c:42693 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T19:13:30,993 INFO [RS:1;692b295ad45c:42693 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T19:13:30,993 INFO [RS:1;692b295ad45c:42693 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42693 2024-11-08T19:13:30,994 INFO [regionserver/692b295ad45c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T19:13:30,995 DEBUG [RS:0;692b295ad45c:36007 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/oldWALs 2024-11-08T19:13:30,995 INFO [RS:0;692b295ad45c:36007 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 692b295ad45c%2C36007%2C1731093202744:(num 1731093204811) 2024-11-08T19:13:30,995 DEBUG [RS:0;692b295ad45c:36007 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T19:13:30,996 INFO [RS:0;692b295ad45c:36007 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T19:13:30,996 INFO [RS:0;692b295ad45c:36007 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T19:13:30,996 INFO [RS:0;692b295ad45c:36007 {}] hbase.ChoreService(370): Chore service for: regionserver/692b295ad45c:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-08T19:13:30,996 INFO [RS:0;692b295ad45c:36007 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T19:13:30,996 INFO [regionserver/692b295ad45c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T19:13:30,996 WARN [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] util.Threads(127): sleep interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2374) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T19:13:30,996 WARN [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,rrr,1731093205731.f1496a2d5d6037cbe8d2f7a8f1407184., aborting... java.io.IOException: Failed to report opened region to master: test,rrr,1731093205731.f1496a2d5d6037cbe8d2f7a8f1407184. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T19:13:30,997 ERROR [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2440): ***** ABORTING region server 692b295ad45c,36007,1731093202744: Failed to open region test,rrr,1731093205731.f1496a2d5d6037cbe8d2f7a8f1407184. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,rrr,1731093205731.f1496a2d5d6037cbe8d2f7a8f1407184. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T19:13:30,997 ERROR [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-08T19:13:30,997 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-08T19:13:30,998 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-08T19:13:30,998 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-08T19:13:30,998 INFO [RS:0;692b295ad45c:36007 {}] executor.ExecutorService(108): Executor-13-RS_OPEN_REGION-regionserver/692b295ad45c:0 had [Event #36 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler), Event #37 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler), Event #38 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler)] on shutdown 2024-11-08T19:13:30,999 INFO [RS:0;692b295ad45c:36007 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36007 2024-11-08T19:13:30,999 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-08T19:13:31,000 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T19:13:31,000 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/692b295ad45c,42693,1731093202977 2024-11-08T19:13:31,000 INFO [RS:1;692b295ad45c:42693 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T19:13:31,001 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 472381760 }, "NonHeapMemoryUsage": { "committed": 162922496, "init": 7667712, "max": -1, "used": 160303704 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "692b295ad45c", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 40669, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 132, "ProcessCallTime_min": 0, "ProcessCallTime_max": 29, "ProcessCallTime_mean": 2, "ProcessCallTime_25th_percentile": 7, "ProcessCallTime_median": 14, "ProcessCallTime_75th_percentile": 21, "ProcessCallTime_90th_percentile": 26, "ProcessCallTime_95th_percentile": 27, "ProcessCallTime_98th_percentile": 28, "ProcessCallTime_99th_percentile": 28, "ProcessCallTime_99.9th_percentile": 28, "ProcessCallTime_TimeRangeCount_0-1": 132, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 132, "QueueCallTime_min": 0, "QueueCallTime_max": 16, "QueueCallTime_mean": 1, "QueueCallTime_25th_percentile": 4, "QueueCallTime_median": 8, "QueueCallTime_75th_percentile": 12, "QueueCallTime_90th_percentile": 14, "QueueCallTime_95th_percentile": 15, "QueueCallTime_98th_percentile": 15, "QueueCallTime_99th_percentile": 15, "QueueCallTime_99.9th_percentile": 15, "QueueCallTime_TimeRangeCount_0-1": 132, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 132, "TotalCallTime_min": 0, "TotalCallTime_max": 29, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 7, "TotalCallTime_median": 14, "TotalCallTime_75th_percentile": 21, "TotalCallTime_90th_percentile": 26, "TotalCallTime_95th_percentile": 27, "TotalCallTime_98th_percentile": 28, "TotalCallTime_99th_percentile": 28, "TotalCallTime_99.9th_percentile": 28, "TotalCallTime_TimeRangeCount_0-1": 132, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 132, "ResponseSize_min": 0, "ResponseSize_max": 22854, "ResponseSize_mean": 4500, "ResponseSize_25th_percentile": 5713, "ResponseSize_median": 11427, "ResponseSize_75th_percentile": 17140, "ResponseSize_90th_percentile": 20568, "ResponseSize_95th_percentile": 21711, "ResponseSize_98th_percentile": 22396, "ResponseSize_99th_percentile": 22625, "ResponseSize_99.9th_percentile": 22831, "ResponseSize_SizeRangeCount_0-10": 132, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 132, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 279, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 132, "sentBytes": 459895 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "692b295ad45c", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:56395", "tag.serverName": "692b295ad45c,39647,1731093203061", "tag.clusterId": "c2739604-5628-4fa2-a3e8-33881b328b44", "tag.Context": "regionserver", "tag.Hostname": "692b295ad45c", "regionCount": 9, "storeCount": 12, "hlogFileCount": 2, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 28672, "memStoreHeapSize": 51520, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731093203061, "averageRegionSize": 3185, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 38203, "localBytesRead": 38203, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 18.8, "writeRequestRatePerSecond": 16.400000000000002, "ByteBuffAllocatorHeapAllocationBytes": 39391, "ByteBuffAllocatorPoolAllocationBytes": 465920, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 74, "totalRowActionRequestCount": 176, "readRequestCount": 94, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 82, "rpcGetRequestCount": 2, "rpcFullScanRequestCount": 8, "rpcScanRequestCount": 15, "rpcMultiRequestCount": 1, "rpcMutateRequestCount": 56, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 15, "ScanTime_min": 0, "ScanTime_max": 12, "ScanTime_mean": 1, "ScanTime_25th_percentile": 3, "ScanTime_median": 6, "ScanTime_75th_percentile": 9, "ScanTime_90th_percentile": 10, "ScanTime_95th_percentile": 11, "ScanTime_98th_percentile": 11, "ScanTime_99th_percentile": 11, "ScanTime_99.9th_percentile": 11, "ScanTime_TimeRangeCount_0-1": 15, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 56, "Put_min": 1, "Put_max": 8, "Put_mean": 2, "Put_25th_percentile": 2, "Put_median": 4, "Put_75th_percentile": 6, "Put_90th_percentile": 7, "Put_95th_percentile": 7, "Put_98th_percentile": 7, "Put_99th_percentile": 7, "Put_99.9th_percentile": 7, "Put_TimeRangeCount_0-1": 56, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 1, "PutBatch_min": 11, "PutBatch_max": 11, "PutBatch_mean": 11, "PutBatch_25th_percentile": 11, "PutBatch_median": 11, "PutBatch_75th_percentile": 11, "PutBatch_90th_percentile": 11, "PutBatch_95th_percentile": 11, "PutBatch_98th_percentile": 11, "PutBatch_99th_percentile": 11, "PutBatch_99.9th_percentile": 11, "PutBatch_TimeRangeCount_0-1": 1, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 2, "Get_min": 1, "Get_max": 2, "Get_mean": 1, "Get_25th_percentile": 1, "Get_median": 1, "Get_75th_percentile": 1, "Get_90th_percentile": 1, "Get_95th_percentile": 1, "Get_98th_percentile": 1, "Get_99th_percentile": 1, "Get_99.9th_percentile": 1, "Get_TimeRangeCount_0-1": 2, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 15, "ScanSize_min": 0, "ScanSize_max": 22232, "ScanSize_mean": 4465, "ScanSize_25th_percentile": 5558, "ScanSize_median": 11116, "ScanSize_75th_percentile": 16674, "ScanSize_90th_percentile": 20008, "ScanSize_95th_percentile": 21120, "ScanSize_98th_percentile": 21787, "ScanSize_99th_percentile": 22009, "ScanSize_99.9th_percentile": 22209, "ScanSize_SizeRangeCount_0-10": 15, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-08T19:13:31,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/692b295ad45c,36007,1731093202744 2024-11-08T19:13:31,011 INFO [RS:0;692b295ad45c:36007 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T19:13:31,011 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [692b295ad45c,42693,1731093202977] 2024-11-08T19:13:31,032 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/692b295ad45c,42693,1731093202977 already deleted, retry=false 2024-11-08T19:13:31,032 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 692b295ad45c,42693,1731093202977 expired; onlineServers=1 2024-11-08T19:13:31,032 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [692b295ad45c,36007,1731093202744] 2024-11-08T19:13:31,042 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/692b295ad45c,36007,1731093202744 already deleted, retry=false 2024-11-08T19:13:31,042 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 692b295ad45c,36007,1731093202744 expired; onlineServers=0 2024-11-08T19:13:31,043 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '692b295ad45c,39509,1731093201567' ***** 2024-11-08T19:13:31,043 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-08T19:13:31,043 INFO [M:0;692b295ad45c:39509 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T19:13:31,043 INFO [M:0;692b295ad45c:39509 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T19:13:31,043 DEBUG [M:0;692b295ad45c:39509 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-08T19:13:31,043 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-08T19:13:31,043 DEBUG [M:0;692b295ad45c:39509 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-08T19:13:31,043 DEBUG [master/692b295ad45c:0:becomeActiveMaster-HFileCleaner.large.0-1731093204428 {}] cleaner.HFileCleaner(306): Exit Thread[master/692b295ad45c:0:becomeActiveMaster-HFileCleaner.large.0-1731093204428,5,FailOnTimeoutGroup] 2024-11-08T19:13:31,043 DEBUG [master/692b295ad45c:0:becomeActiveMaster-HFileCleaner.small.0-1731093204430 {}] cleaner.HFileCleaner(306): Exit Thread[master/692b295ad45c:0:becomeActiveMaster-HFileCleaner.small.0-1731093204430,5,FailOnTimeoutGroup] 2024-11-08T19:13:31,043 INFO [M:0;692b295ad45c:39509 {}] hbase.ChoreService(370): Chore service for: master/692b295ad45c:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-08T19:13:31,044 INFO [M:0;692b295ad45c:39509 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T19:13:31,044 DEBUG [M:0;692b295ad45c:39509 {}] master.HMaster(1795): Stopping service threads 2024-11-08T19:13:31,044 INFO [M:0;692b295ad45c:39509 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-08T19:13:31,044 INFO [M:0;692b295ad45c:39509 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:13:31,044 INFO [M:0;692b295ad45c:39509 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T19:13:31,044 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-08T19:13:31,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-08T19:13:31,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T19:13:31,053 DEBUG [M:0;692b295ad45c:39509 {}] zookeeper.ZKUtil(347): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-08T19:13:31,053 WARN [M:0;692b295ad45c:39509 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-08T19:13:31,054 INFO [M:0;692b295ad45c:39509 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/.lastflushedseqids 2024-11-08T19:13:31,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741870_1046 (size=1) 2024-11-08T19:13:31,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741870_1046 (size=1) 2024-11-08T19:13:31,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741870_1046 (size=1) 2024-11-08T19:13:31,065 INFO [M:0;692b295ad45c:39509 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T19:13:31,065 INFO [M:0;692b295ad45c:39509 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-08T19:13:31,065 DEBUG [M:0;692b295ad45c:39509 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T19:13:31,065 INFO [M:0;692b295ad45c:39509 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T19:13:31,065 DEBUG [M:0;692b295ad45c:39509 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T19:13:31,065 DEBUG [M:0;692b295ad45c:39509 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T19:13:31,065 DEBUG [M:0;692b295ad45c:39509 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T19:13:31,065 INFO [M:0;692b295ad45c:39509 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=248.11 KB heapSize=301.50 KB 2024-11-08T19:13:31,081 DEBUG [M:0;692b295ad45c:39509 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/128d286c40dd4e589030baacc453c09b is 82, key is hbase:meta,,1/info:regioninfo/1731093209273/Put/seqid=0 2024-11-08T19:13:31,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741871_1047 (size=6071) 2024-11-08T19:13:31,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741871_1047 (size=6071) 2024-11-08T19:13:31,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741871_1047 (size=6071) 2024-11-08T19:13:31,089 INFO [M:0;692b295ad45c:39509 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/128d286c40dd4e589030baacc453c09b 2024-11-08T19:13:31,112 DEBUG [M:0;692b295ad45c:39509 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6a2692f393eb44fbab6027800d8d14af is 2990, key is \x00\x00\x00\x00\x00\x00\x009/proc:d/1731093210741/Put/seqid=0 2024-11-08T19:13:31,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741872_1048 (size=12801) 2024-11-08T19:13:31,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741872_1048 (size=12801) 2024-11-08T19:13:31,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741872_1048 (size=12801) 2024-11-08T19:13:31,119 INFO [M:0;692b295ad45c:39509 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=246.84 KB at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6a2692f393eb44fbab6027800d8d14af 2024-11-08T19:13:31,121 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T19:13:31,121 DEBUG [pool-69-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42693-0x1011bddfb000002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T19:13:31,121 INFO [RS:1;692b295ad45c:42693 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T19:13:31,121 INFO [RS:1;692b295ad45c:42693 {}] regionserver.HRegionServer(1031): Exiting; stopping=692b295ad45c,42693,1731093202977; zookeeper connection closed. 2024-11-08T19:13:31,122 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@728e5b8b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@728e5b8b 2024-11-08T19:13:31,132 INFO [RS:0;692b295ad45c:36007 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T19:13:31,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T19:13:31,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36007-0x1011bddfb000001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T19:13:31,132 INFO [RS:0;692b295ad45c:36007 {}] regionserver.HRegionServer(1031): Exiting; stopping=692b295ad45c,36007,1731093202744; zookeeper connection closed. 2024-11-08T19:13:31,133 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@32e11ee7 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@32e11ee7 2024-11-08T19:13:31,133 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-08T19:13:31,144 DEBUG [M:0;692b295ad45c:39509 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/dcae4b725a564134be73a4bb86378523 is 69, key is 692b295ad45c,36007,1731093202744/rs:state/1731093204507/Put/seqid=0 2024-11-08T19:13:31,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741873_1049 (size=5447) 2024-11-08T19:13:31,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741873_1049 (size=5447) 2024-11-08T19:13:31,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741873_1049 (size=5447) 2024-11-08T19:13:31,153 INFO [M:0;692b295ad45c:39509 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/dcae4b725a564134be73a4bb86378523 2024-11-08T19:13:31,160 INFO [M:0;692b295ad45c:39509 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for dcae4b725a564134be73a4bb86378523 2024-11-08T19:13:31,180 DEBUG [M:0;692b295ad45c:39509 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/f53905eef2f74286b4f565391ab1f336 is 52, key is load_balancer_on/state:d/1731093208061/Put/seqid=0 2024-11-08T19:13:31,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741874_1050 (size=5058) 2024-11-08T19:13:31,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741874_1050 (size=5058) 2024-11-08T19:13:31,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741874_1050 (size=5058) 2024-11-08T19:13:31,188 INFO [M:0;692b295ad45c:39509 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/f53905eef2f74286b4f565391ab1f336 2024-11-08T19:13:31,196 DEBUG [M:0;692b295ad45c:39509 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/128d286c40dd4e589030baacc453c09b as hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/128d286c40dd4e589030baacc453c09b 2024-11-08T19:13:31,202 INFO [M:0;692b295ad45c:39509 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/128d286c40dd4e589030baacc453c09b, entries=14, sequenceid=569, filesize=5.9 K 2024-11-08T19:13:31,203 DEBUG [M:0;692b295ad45c:39509 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6a2692f393eb44fbab6027800d8d14af as hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6a2692f393eb44fbab6027800d8d14af 2024-11-08T19:13:31,210 INFO [M:0;692b295ad45c:39509 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6a2692f393eb44fbab6027800d8d14af, entries=79, sequenceid=569, filesize=12.5 K 2024-11-08T19:13:31,211 DEBUG [M:0;692b295ad45c:39509 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/dcae4b725a564134be73a4bb86378523 as hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/dcae4b725a564134be73a4bb86378523 2024-11-08T19:13:31,218 INFO [M:0;692b295ad45c:39509 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for dcae4b725a564134be73a4bb86378523 2024-11-08T19:13:31,218 INFO [M:0;692b295ad45c:39509 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/dcae4b725a564134be73a4bb86378523, entries=3, sequenceid=569, filesize=5.3 K 2024-11-08T19:13:31,219 DEBUG [M:0;692b295ad45c:39509 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/f53905eef2f74286b4f565391ab1f336 as hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/f53905eef2f74286b4f565391ab1f336 2024-11-08T19:13:31,228 INFO [M:0;692b295ad45c:39509 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33621/user/jenkins/test-data/c8b8ba33-7c72-14b8-20d1-a6effd52479f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/f53905eef2f74286b4f565391ab1f336, entries=1, sequenceid=569, filesize=4.9 K 2024-11-08T19:13:31,229 INFO [M:0;692b295ad45c:39509 {}] regionserver.HRegion(3140): Finished flush of dataSize ~248.11 KB/254068, heapSize ~301.44 KB/308672, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 164ms, sequenceid=569, compaction requested=false 2024-11-08T19:13:31,231 INFO [M:0;692b295ad45c:39509 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T19:13:31,231 DEBUG [M:0;692b295ad45c:39509 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731093211065Disabling compacts and flushes for region at 1731093211065Disabling writes for close at 1731093211065Obtaining lock to block concurrent updates at 1731093211065Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731093211065Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=254068, getHeapSize=308672, getOffHeapSize=0, getCellsCount=708 at 1731093211066 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731093211066Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731093211066Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731093211081 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731093211081Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731093211095 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731093211111 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731093211111Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731093211125 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731093211143 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731093211144 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731093211160 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731093211180 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731093211180Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@45bde947: reopening flushed file at 1731093211195 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@20975cc4: reopening flushed file at 1731093211202 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21f6f057: reopening flushed file at 1731093211210 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33a00691: reopening flushed file at 1731093211218 (+8 ms)Finished flush of dataSize ~248.11 KB/254068, heapSize ~301.44 KB/308672, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 164ms, sequenceid=569, compaction requested=false at 1731093211229 (+11 ms)Writing region close event to WAL at 1731093211231 (+2 ms)Closed at 1731093211231 2024-11-08T19:13:31,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42249 is added to blk_1073741830_1006 (size=292062) 2024-11-08T19:13:31,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46657 is added to blk_1073741830_1006 (size=292062) 2024-11-08T19:13:31,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741830_1006 (size=292062) 2024-11-08T19:13:31,235 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T19:13:31,235 INFO [M:0;692b295ad45c:39509 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-08T19:13:31,235 INFO [M:0;692b295ad45c:39509 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39509 2024-11-08T19:13:31,235 INFO [M:0;692b295ad45c:39509 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T19:13:31,343 INFO [M:0;692b295ad45c:39509 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T19:13:31,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T19:13:31,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39509-0x1011bddfb000000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T19:13:31,346 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3c1f701{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T19:13:31,348 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3d599bb4{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T19:13:31,348 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T19:13:31,349 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@417efacb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T19:13:31,349 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@290e2eec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/hadoop.log.dir/,STOPPED} 2024-11-08T19:13:31,351 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-08T19:13:31,351 WARN [BP-585874096-172.17.0.2-1731093195986 heartbeating to localhost/127.0.0.1:33621 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-08T19:13:31,351 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-08T19:13:31,351 WARN [BP-585874096-172.17.0.2-1731093195986 heartbeating to localhost/127.0.0.1:33621 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-585874096-172.17.0.2-1731093195986 (Datanode Uuid e5fad3ad-0de2-4a49-8a13-2c9609c5c7b8) service to localhost/127.0.0.1:33621 2024-11-08T19:13:31,353 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/cluster_79c0c4f2-2bfc-2c08-96de-113f2fd3fb9c/data/data5/current/BP-585874096-172.17.0.2-1731093195986 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T19:13:31,353 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/cluster_79c0c4f2-2bfc-2c08-96de-113f2fd3fb9c/data/data6/current/BP-585874096-172.17.0.2-1731093195986 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T19:13:31,354 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-08T19:13:31,355 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@34fcb585{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T19:13:31,356 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@389bb6f7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T19:13:31,356 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T19:13:31,356 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1b0a32ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T19:13:31,356 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@29652cf4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/hadoop.log.dir/,STOPPED} 2024-11-08T19:13:31,357 WARN [BP-585874096-172.17.0.2-1731093195986 heartbeating to localhost/127.0.0.1:33621 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-08T19:13:31,357 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-08T19:13:31,357 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-08T19:13:31,357 WARN [BP-585874096-172.17.0.2-1731093195986 heartbeating to localhost/127.0.0.1:33621 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-585874096-172.17.0.2-1731093195986 (Datanode Uuid df5501d9-e532-471c-b303-aebd945d69ac) service to localhost/127.0.0.1:33621 2024-11-08T19:13:31,358 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/cluster_79c0c4f2-2bfc-2c08-96de-113f2fd3fb9c/data/data3/current/BP-585874096-172.17.0.2-1731093195986 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T19:13:31,358 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/cluster_79c0c4f2-2bfc-2c08-96de-113f2fd3fb9c/data/data4/current/BP-585874096-172.17.0.2-1731093195986 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T19:13:31,359 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-08T19:13:31,365 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@79b30fb0{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T19:13:31,365 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@548cd725{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T19:13:31,365 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T19:13:31,366 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@474d518f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T19:13:31,366 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3a300759{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/hadoop.log.dir/,STOPPED} 2024-11-08T19:13:31,367 WARN [BP-585874096-172.17.0.2-1731093195986 heartbeating to localhost/127.0.0.1:33621 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-08T19:13:31,367 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-08T19:13:31,367 WARN [BP-585874096-172.17.0.2-1731093195986 heartbeating to localhost/127.0.0.1:33621 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-585874096-172.17.0.2-1731093195986 (Datanode Uuid 3a969f21-2fc4-4416-ac5c-2994a3eb4f53) service to localhost/127.0.0.1:33621 2024-11-08T19:13:31,367 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-08T19:13:31,367 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/cluster_79c0c4f2-2bfc-2c08-96de-113f2fd3fb9c/data/data1/current/BP-585874096-172.17.0.2-1731093195986 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T19:13:31,368 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/cluster_79c0c4f2-2bfc-2c08-96de-113f2fd3fb9c/data/data2/current/BP-585874096-172.17.0.2-1731093195986 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T19:13:31,368 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-08T19:13:31,379 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@64418798{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-08T19:13:31,380 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T19:13:31,380 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T19:13:31,380 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T19:13:31,380 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/70e75a57-6ecf-e633-8958-097fbd53b6bd/hadoop.log.dir/,STOPPED} 2024-11-08T19:13:31,389 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-08T19:13:31,438 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down