2024-11-26 21:49:44,969 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-26 21:49:44,989 main DEBUG Took 0.016545 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-26 21:49:44,989 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-26 21:49:44,990 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-26 21:49:44,992 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-26 21:49:44,994 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 21:49:45,006 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-26 21:49:45,024 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 21:49:45,026 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 21:49:45,027 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 21:49:45,027 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 21:49:45,028 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 21:49:45,028 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 21:49:45,030 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 21:49:45,030 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 21:49:45,031 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 21:49:45,032 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 21:49:45,033 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 21:49:45,034 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 21:49:45,036 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 21:49:45,037 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 21:49:45,038 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 21:49:45,039 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 21:49:45,039 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 21:49:45,040 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 21:49:45,041 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 21:49:45,042 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 21:49:45,042 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 21:49:45,043 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 21:49:45,044 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 21:49:45,044 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 21:49:45,045 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 21:49:45,046 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-26 21:49:45,048 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 21:49:45,049 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-26 21:49:45,051 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-26 21:49:45,052 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-26 21:49:45,054 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-26 21:49:45,055 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-26 21:49:45,068 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-26 21:49:45,073 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-26 21:49:45,075 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-26 21:49:45,076 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-26 21:49:45,076 main DEBUG createAppenders(={Console}) 2024-11-26 21:49:45,078 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-11-26 21:49:45,080 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-26 21:49:45,080 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-11-26 21:49:45,081 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-26 21:49:45,082 main DEBUG OutputStream closed 2024-11-26 21:49:45,082 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-26 21:49:45,083 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-26 21:49:45,083 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-11-26 21:49:45,185 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-26 21:49:45,189 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-26 21:49:45,190 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-26 21:49:45,191 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-26 21:49:45,192 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-26 21:49:45,193 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-26 21:49:45,193 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-26 21:49:45,194 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-26 21:49:45,195 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-26 21:49:45,195 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-26 21:49:45,196 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-26 21:49:45,196 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-26 21:49:45,197 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-26 21:49:45,198 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-26 21:49:45,198 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-26 21:49:45,199 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-26 21:49:45,199 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-26 21:49:45,200 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-26 21:49:45,202 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26 21:49:45,203 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-11-26 21:49:45,203 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-26 21:49:45,204 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-11-26T21:49:45,221 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-11-26 21:49:45,226 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-26 21:49:45,226 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26T21:49:45,551 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5 2024-11-26T21:49:45,568 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-26T21:49:45,583 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/cluster_626ec74d-0f86-b49f-b08d-4173446b08f3, deleteOnExit=true 2024-11-26T21:49:45,584 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-26T21:49:45,585 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/test.cache.data in system properties and HBase conf 2024-11-26T21:49:45,586 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/hadoop.tmp.dir in system properties and HBase conf 2024-11-26T21:49:45,586 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/hadoop.log.dir in system properties and HBase conf 2024-11-26T21:49:45,587 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-26T21:49:45,588 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-26T21:49:45,588 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-26T21:49:45,673 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-26T21:49:45,762 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T21:49:45,766 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-26T21:49:45,766 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-26T21:49:45,767 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-26T21:49:45,767 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T21:49:45,768 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-26T21:49:45,768 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-26T21:49:45,768 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T21:49:45,769 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T21:49:45,769 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-26T21:49:45,770 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/nfs.dump.dir in system properties and HBase conf 2024-11-26T21:49:45,770 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/java.io.tmpdir in system properties and HBase conf 2024-11-26T21:49:45,770 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T21:49:45,771 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-26T21:49:45,771 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-26T21:49:46,550 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-26T21:49:46,628 INFO [Time-limited test {}] log.Log(170): Logging initialized @2472ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-26T21:49:46,709 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T21:49:46,767 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T21:49:46,786 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T21:49:46,786 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T21:49:46,787 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-26T21:49:46,798 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T21:49:46,801 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@265b24f1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/hadoop.log.dir/,AVAILABLE} 2024-11-26T21:49:46,802 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@76beb053{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T21:49:46,988 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@376fa8f3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/java.io.tmpdir/jetty-localhost-37291-hadoop-hdfs-3_4_1-tests_jar-_-any-9641751732693900864/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T21:49:46,999 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@672b88ea{HTTP/1.1, (http/1.1)}{localhost:37291} 2024-11-26T21:49:47,000 INFO [Time-limited test {}] server.Server(415): Started @2844ms 2024-11-26T21:49:47,322 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T21:49:47,331 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T21:49:47,332 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T21:49:47,332 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T21:49:47,333 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-26T21:49:47,333 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@739a104e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/hadoop.log.dir/,AVAILABLE} 2024-11-26T21:49:47,334 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1c5d2af1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T21:49:47,435 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2226d6a6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/java.io.tmpdir/jetty-localhost-45983-hadoop-hdfs-3_4_1-tests_jar-_-any-10104224838511305482/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T21:49:47,436 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1da1f773{HTTP/1.1, (http/1.1)}{localhost:45983} 2024-11-26T21:49:47,436 INFO [Time-limited test {}] server.Server(415): Started @3280ms 2024-11-26T21:49:47,488 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T21:49:47,595 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T21:49:47,600 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T21:49:47,605 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T21:49:47,606 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T21:49:47,606 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-26T21:49:47,607 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@34a3e1ff{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/hadoop.log.dir/,AVAILABLE} 2024-11-26T21:49:47,608 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@12f16529{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T21:49:47,713 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7cfccf27{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/java.io.tmpdir/jetty-localhost-38715-hadoop-hdfs-3_4_1-tests_jar-_-any-248575375522738041/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T21:49:47,714 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6c41bbb7{HTTP/1.1, (http/1.1)}{localhost:38715} 2024-11-26T21:49:47,714 INFO [Time-limited test {}] server.Server(415): Started @3559ms 2024-11-26T21:49:47,717 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T21:49:47,771 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T21:49:47,779 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T21:49:47,789 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T21:49:47,789 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T21:49:47,789 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-26T21:49:47,791 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11ebaa12{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/hadoop.log.dir/,AVAILABLE} 2024-11-26T21:49:47,792 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@52cf89e6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T21:49:47,883 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/cluster_626ec74d-0f86-b49f-b08d-4173446b08f3/data/data2/current/BP-655727968-172.17.0.2-1732657786342/current, will proceed with Du for space computation calculation, 2024-11-26T21:49:47,883 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/cluster_626ec74d-0f86-b49f-b08d-4173446b08f3/data/data4/current/BP-655727968-172.17.0.2-1732657786342/current, will proceed with Du for space computation calculation, 2024-11-26T21:49:47,883 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/cluster_626ec74d-0f86-b49f-b08d-4173446b08f3/data/data1/current/BP-655727968-172.17.0.2-1732657786342/current, will proceed with Du for space computation calculation, 2024-11-26T21:49:47,883 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/cluster_626ec74d-0f86-b49f-b08d-4173446b08f3/data/data3/current/BP-655727968-172.17.0.2-1732657786342/current, will proceed with Du for space computation calculation, 2024-11-26T21:49:47,920 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T21:49:47,921 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T21:49:47,927 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@67ddfd25{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/java.io.tmpdir/jetty-localhost-46507-hadoop-hdfs-3_4_1-tests_jar-_-any-5515712820206579188/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T21:49:47,928 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@30e38263{HTTP/1.1, (http/1.1)}{localhost:46507} 2024-11-26T21:49:47,929 INFO [Time-limited test {}] server.Server(415): Started @3773ms 2024-11-26T21:49:47,931 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T21:49:47,997 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x21754ab6208aeeea with lease ID 0xd32d5dae4bd62a1c: Processing first storage report for DS-aed5038f-ce60-4629-a808-1e2f6bde72d8 from datanode DatanodeRegistration(127.0.0.1:39537, datanodeUuid=6fd10630-2eb6-418e-a56d-198777b09dd3, infoPort=45705, infoSecurePort=0, ipcPort=41489, storageInfo=lv=-57;cid=testClusterID;nsid=62704618;c=1732657786342) 2024-11-26T21:49:47,999 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x21754ab6208aeeea with lease ID 0xd32d5dae4bd62a1c: from storage DS-aed5038f-ce60-4629-a808-1e2f6bde72d8 node DatanodeRegistration(127.0.0.1:39537, datanodeUuid=6fd10630-2eb6-418e-a56d-198777b09dd3, infoPort=45705, infoSecurePort=0, ipcPort=41489, storageInfo=lv=-57;cid=testClusterID;nsid=62704618;c=1732657786342), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-26T21:49:48,000 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x99e2151c77a0b269 with lease ID 0xd32d5dae4bd62a1d: Processing first storage report for DS-4fbda298-e9f3-436b-9808-ab5ec6eece62 from datanode DatanodeRegistration(127.0.0.1:40345, datanodeUuid=b26d6d41-1899-4034-82ea-25ab341b922e, infoPort=36805, infoSecurePort=0, ipcPort=36135, storageInfo=lv=-57;cid=testClusterID;nsid=62704618;c=1732657786342) 2024-11-26T21:49:48,000 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x99e2151c77a0b269 with lease ID 0xd32d5dae4bd62a1d: from storage DS-4fbda298-e9f3-436b-9808-ab5ec6eece62 node DatanodeRegistration(127.0.0.1:40345, datanodeUuid=b26d6d41-1899-4034-82ea-25ab341b922e, infoPort=36805, infoSecurePort=0, ipcPort=36135, storageInfo=lv=-57;cid=testClusterID;nsid=62704618;c=1732657786342), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-26T21:49:48,000 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x21754ab6208aeeea with lease ID 0xd32d5dae4bd62a1c: Processing first storage report for DS-024a32bc-ad5c-4fa5-a07f-ce41d5f72238 from datanode DatanodeRegistration(127.0.0.1:39537, datanodeUuid=6fd10630-2eb6-418e-a56d-198777b09dd3, infoPort=45705, infoSecurePort=0, ipcPort=41489, storageInfo=lv=-57;cid=testClusterID;nsid=62704618;c=1732657786342) 2024-11-26T21:49:48,001 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x21754ab6208aeeea with lease ID 0xd32d5dae4bd62a1c: from storage DS-024a32bc-ad5c-4fa5-a07f-ce41d5f72238 node DatanodeRegistration(127.0.0.1:39537, datanodeUuid=6fd10630-2eb6-418e-a56d-198777b09dd3, infoPort=45705, infoSecurePort=0, ipcPort=41489, storageInfo=lv=-57;cid=testClusterID;nsid=62704618;c=1732657786342), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T21:49:48,001 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x99e2151c77a0b269 with lease ID 0xd32d5dae4bd62a1d: Processing first storage report for DS-ba9fbdc1-f897-452c-ba22-8e63ebffb49a from datanode DatanodeRegistration(127.0.0.1:40345, datanodeUuid=b26d6d41-1899-4034-82ea-25ab341b922e, infoPort=36805, infoSecurePort=0, ipcPort=36135, storageInfo=lv=-57;cid=testClusterID;nsid=62704618;c=1732657786342) 2024-11-26T21:49:48,001 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x99e2151c77a0b269 with lease ID 0xd32d5dae4bd62a1d: from storage DS-ba9fbdc1-f897-452c-ba22-8e63ebffb49a node DatanodeRegistration(127.0.0.1:40345, datanodeUuid=b26d6d41-1899-4034-82ea-25ab341b922e, infoPort=36805, infoSecurePort=0, ipcPort=36135, storageInfo=lv=-57;cid=testClusterID;nsid=62704618;c=1732657786342), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T21:49:48,037 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/cluster_626ec74d-0f86-b49f-b08d-4173446b08f3/data/data5/current/BP-655727968-172.17.0.2-1732657786342/current, will proceed with Du for space computation calculation, 2024-11-26T21:49:48,038 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/cluster_626ec74d-0f86-b49f-b08d-4173446b08f3/data/data6/current/BP-655727968-172.17.0.2-1732657786342/current, will proceed with Du for space computation calculation, 2024-11-26T21:49:48,066 WARN [Thread-129 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T21:49:48,073 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe71bf0bf09427b71 with lease ID 0xd32d5dae4bd62a1e: Processing first storage report for DS-7dc835d1-8731-44e2-b974-f85d5c6f47f1 from datanode DatanodeRegistration(127.0.0.1:43659, datanodeUuid=72e127b1-5812-444a-9dfb-62013621bbf8, infoPort=36781, infoSecurePort=0, ipcPort=41815, storageInfo=lv=-57;cid=testClusterID;nsid=62704618;c=1732657786342) 2024-11-26T21:49:48,073 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe71bf0bf09427b71 with lease ID 0xd32d5dae4bd62a1e: from storage DS-7dc835d1-8731-44e2-b974-f85d5c6f47f1 node DatanodeRegistration(127.0.0.1:43659, datanodeUuid=72e127b1-5812-444a-9dfb-62013621bbf8, infoPort=36781, infoSecurePort=0, ipcPort=41815, storageInfo=lv=-57;cid=testClusterID;nsid=62704618;c=1732657786342), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-26T21:49:48,073 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe71bf0bf09427b71 with lease ID 0xd32d5dae4bd62a1e: Processing first storage report for DS-303c33e8-259a-4fba-bff9-5c3da08282c8 from datanode DatanodeRegistration(127.0.0.1:43659, datanodeUuid=72e127b1-5812-444a-9dfb-62013621bbf8, infoPort=36781, infoSecurePort=0, ipcPort=41815, storageInfo=lv=-57;cid=testClusterID;nsid=62704618;c=1732657786342) 2024-11-26T21:49:48,073 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe71bf0bf09427b71 with lease ID 0xd32d5dae4bd62a1e: from storage DS-303c33e8-259a-4fba-bff9-5c3da08282c8 node DatanodeRegistration(127.0.0.1:43659, datanodeUuid=72e127b1-5812-444a-9dfb-62013621bbf8, infoPort=36781, infoSecurePort=0, ipcPort=41815, storageInfo=lv=-57;cid=testClusterID;nsid=62704618;c=1732657786342), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T21:49:48,280 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5 2024-11-26T21:49:48,351 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/cluster_626ec74d-0f86-b49f-b08d-4173446b08f3/zookeeper_0, clientPort=53387, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/cluster_626ec74d-0f86-b49f-b08d-4173446b08f3/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/cluster_626ec74d-0f86-b49f-b08d-4173446b08f3/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-26T21:49:48,362 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53387 2024-11-26T21:49:48,374 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T21:49:48,377 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T21:49:48,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741825_1001 (size=7) 2024-11-26T21:49:48,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741825_1001 (size=7) 2024-11-26T21:49:48,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741825_1001 (size=7) 2024-11-26T21:49:49,039 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657 with version=8 2024-11-26T21:49:49,039 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/hbase-staging 2024-11-26T21:49:49,120 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-26T21:49:49,317 INFO [Time-limited test {}] client.ConnectionUtils(128): master/828b3f090c91:0 server-side Connection retries=45 2024-11-26T21:49:49,325 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T21:49:49,326 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T21:49:49,330 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T21:49:49,330 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T21:49:49,330 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T21:49:49,463 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-26T21:49:49,523 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-26T21:49:49,531 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-26T21:49:49,534 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T21:49:49,556 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 85896 (auto-detected) 2024-11-26T21:49:49,557 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-26T21:49:49,574 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44203 2024-11-26T21:49:49,594 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44203 connecting to ZooKeeper ensemble=127.0.0.1:53387 2024-11-26T21:49:49,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:442030x0, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T21:49:49,621 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44203-0x1007e03121b0000 connected 2024-11-26T21:49:49,646 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T21:49:49,650 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T21:49:49,660 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T21:49:49,663 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657, hbase.cluster.distributed=false 2024-11-26T21:49:49,685 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T21:49:49,690 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44203 2024-11-26T21:49:49,691 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44203 2024-11-26T21:49:49,691 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44203 2024-11-26T21:49:49,692 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44203 2024-11-26T21:49:49,692 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44203 2024-11-26T21:49:49,785 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/828b3f090c91:0 server-side Connection retries=45 2024-11-26T21:49:49,787 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T21:49:49,787 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T21:49:49,787 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T21:49:49,787 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T21:49:49,787 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T21:49:49,790 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T21:49:49,793 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T21:49:49,793 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46545 2024-11-26T21:49:49,795 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46545 connecting to ZooKeeper ensemble=127.0.0.1:53387 2024-11-26T21:49:49,796 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T21:49:49,798 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T21:49:49,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:465450x0, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T21:49:49,804 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46545-0x1007e03121b0001, quorum=127.0.0.1:53387, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T21:49:49,804 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46545-0x1007e03121b0001 connected 2024-11-26T21:49:49,807 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T21:49:49,814 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T21:49:49,816 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46545-0x1007e03121b0001, quorum=127.0.0.1:53387, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T21:49:49,821 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46545-0x1007e03121b0001, quorum=127.0.0.1:53387, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T21:49:49,822 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46545 2024-11-26T21:49:49,822 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46545 2024-11-26T21:49:49,823 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46545 2024-11-26T21:49:49,823 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46545 2024-11-26T21:49:49,824 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46545 2024-11-26T21:49:49,839 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/828b3f090c91:0 server-side Connection retries=45 2024-11-26T21:49:49,840 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T21:49:49,840 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T21:49:49,840 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T21:49:49,840 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T21:49:49,841 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T21:49:49,841 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T21:49:49,841 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T21:49:49,842 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40359 2024-11-26T21:49:49,844 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40359 connecting to ZooKeeper ensemble=127.0.0.1:53387 2024-11-26T21:49:49,845 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T21:49:49,848 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T21:49:49,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:403590x0, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T21:49:49,854 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40359-0x1007e03121b0002 connected 2024-11-26T21:49:49,854 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T21:49:49,855 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T21:49:49,856 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T21:49:49,857 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T21:49:49,858 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T21:49:49,859 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40359 2024-11-26T21:49:49,859 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40359 2024-11-26T21:49:49,861 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40359 2024-11-26T21:49:49,862 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40359 2024-11-26T21:49:49,862 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40359 2024-11-26T21:49:49,878 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/828b3f090c91:0 server-side Connection retries=45 2024-11-26T21:49:49,878 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T21:49:49,878 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T21:49:49,878 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T21:49:49,879 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T21:49:49,879 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T21:49:49,879 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T21:49:49,879 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T21:49:49,880 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42273 2024-11-26T21:49:49,882 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42273 connecting to ZooKeeper ensemble=127.0.0.1:53387 2024-11-26T21:49:49,883 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T21:49:49,885 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T21:49:49,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:422730x0, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T21:49:49,891 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:422730x0, quorum=127.0.0.1:53387, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T21:49:49,891 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42273-0x1007e03121b0003 connected 2024-11-26T21:49:49,892 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T21:49:49,892 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T21:49:49,893 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T21:49:49,895 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T21:49:49,897 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42273 2024-11-26T21:49:49,898 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42273 2024-11-26T21:49:49,898 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42273 2024-11-26T21:49:49,901 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42273 2024-11-26T21:49:49,902 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42273 2024-11-26T21:49:49,916 DEBUG [M:0;828b3f090c91:44203 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;828b3f090c91:44203 2024-11-26T21:49:49,917 INFO [master/828b3f090c91:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/828b3f090c91,44203,1732657789166 2024-11-26T21:49:49,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46545-0x1007e03121b0001, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T21:49:49,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T21:49:49,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T21:49:49,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T21:49:49,925 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/828b3f090c91,44203,1732657789166 2024-11-26T21:49:49,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T21:49:49,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T21:49:49,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46545-0x1007e03121b0001, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T21:49:49,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:49:49,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:49:49,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46545-0x1007e03121b0001, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:49:49,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:49:49,952 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-26T21:49:49,954 INFO [master/828b3f090c91:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/828b3f090c91,44203,1732657789166 from backup master directory 2024-11-26T21:49:49,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/828b3f090c91,44203,1732657789166 2024-11-26T21:49:49,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T21:49:49,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46545-0x1007e03121b0001, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T21:49:49,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T21:49:49,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T21:49:49,960 WARN [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T21:49:49,960 INFO [master/828b3f090c91:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=828b3f090c91,44203,1732657789166 2024-11-26T21:49:49,963 INFO [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-26T21:49:49,965 INFO [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-26T21:49:50,028 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/hbase.id] with ID: 6d98cb23-728a-4b03-88ab-90833a5d474e 2024-11-26T21:49:50,028 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/.tmp/hbase.id 2024-11-26T21:49:50,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741826_1002 (size=42) 2024-11-26T21:49:50,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741826_1002 (size=42) 2024-11-26T21:49:50,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741826_1002 (size=42) 2024-11-26T21:49:50,044 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/.tmp/hbase.id]:[hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/hbase.id] 2024-11-26T21:49:50,093 INFO [master/828b3f090c91:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T21:49:50,098 INFO [master/828b3f090c91:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-26T21:49:50,116 INFO [master/828b3f090c91:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-26T21:49:50,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46545-0x1007e03121b0001, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:49:50,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:49:50,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:49:50,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:49:50,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741827_1003 (size=196) 2024-11-26T21:49:50,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741827_1003 (size=196) 2024-11-26T21:49:50,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741827_1003 (size=196) 2024-11-26T21:49:50,155 INFO [master/828b3f090c91:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T21:49:50,156 INFO [master/828b3f090c91:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-26T21:49:50,170 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T21:49:50,173 INFO [master/828b3f090c91:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T21:49:50,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741828_1004 (size=1189) 2024-11-26T21:49:50,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741828_1004 (size=1189) 2024-11-26T21:49:50,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741828_1004 (size=1189) 2024-11-26T21:49:50,222 INFO [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/MasterData/data/master/store 2024-11-26T21:49:50,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741829_1005 (size=34) 2024-11-26T21:49:50,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741829_1005 (size=34) 2024-11-26T21:49:50,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741829_1005 (size=34) 2024-11-26T21:49:50,250 INFO [master/828b3f090c91:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-26T21:49:50,253 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:50,255 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T21:49:50,255 INFO [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T21:49:50,255 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T21:49:50,257 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T21:49:50,257 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T21:49:50,257 INFO [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T21:49:50,259 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732657790254Disabling compacts and flushes for region at 1732657790254Disabling writes for close at 1732657790257 (+3 ms)Writing region close event to WAL at 1732657790257Closed at 1732657790257 2024-11-26T21:49:50,261 WARN [master/828b3f090c91:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/MasterData/data/master/store/.initializing 2024-11-26T21:49:50,261 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/MasterData/WALs/828b3f090c91,44203,1732657789166 2024-11-26T21:49:50,269 INFO [master/828b3f090c91:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T21:49:50,283 INFO [master/828b3f090c91:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=828b3f090c91%2C44203%2C1732657789166, suffix=, logDir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/MasterData/WALs/828b3f090c91,44203,1732657789166, archiveDir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/MasterData/oldWALs, maxLogs=10 2024-11-26T21:49:50,306 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/MasterData/WALs/828b3f090c91,44203,1732657789166/828b3f090c91%2C44203%2C1732657789166.1732657790287, exclude list is [], retry=0 2024-11-26T21:49:50,322 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40345,DS-4fbda298-e9f3-436b-9808-ab5ec6eece62,DISK] 2024-11-26T21:49:50,322 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43659,DS-7dc835d1-8731-44e2-b974-f85d5c6f47f1,DISK] 2024-11-26T21:49:50,322 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39537,DS-aed5038f-ce60-4629-a808-1e2f6bde72d8,DISK] 2024-11-26T21:49:50,325 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-26T21:49:50,362 INFO [master/828b3f090c91:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/MasterData/WALs/828b3f090c91,44203,1732657789166/828b3f090c91%2C44203%2C1732657789166.1732657790287 2024-11-26T21:49:50,363 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45705:45705),(127.0.0.1/127.0.0.1:36781:36781),(127.0.0.1/127.0.0.1:36805:36805)] 2024-11-26T21:49:50,363 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-26T21:49:50,363 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:50,366 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T21:49:50,367 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T21:49:50,403 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T21:49:50,426 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-26T21:49:50,429 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:50,432 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T21:49:50,433 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T21:49:50,437 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-26T21:49:50,437 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:50,438 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:49:50,438 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T21:49:50,441 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-26T21:49:50,441 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:50,442 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:49:50,442 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T21:49:50,445 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-26T21:49:50,445 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:50,446 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:49:50,446 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T21:49:50,450 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T21:49:50,451 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T21:49:50,456 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T21:49:50,457 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T21:49:50,460 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T21:49:50,464 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T21:49:50,468 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T21:49:50,469 INFO [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59553490, jitterRate=-0.11258384585380554}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T21:49:50,476 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732657790380Initializing all the Stores at 1732657790382 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732657790382Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657790383 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657790383Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657790383Cleaning up temporary data from old regions at 1732657790457 (+74 ms)Region opened successfully at 1732657790476 (+19 ms) 2024-11-26T21:49:50,477 INFO [master/828b3f090c91:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-26T21:49:50,507 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd18405, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=828b3f090c91/172.17.0.2:0 2024-11-26T21:49:50,534 INFO [master/828b3f090c91:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-26T21:49:50,543 INFO [master/828b3f090c91:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T21:49:50,544 INFO [master/828b3f090c91:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T21:49:50,546 INFO [master/828b3f090c91:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T21:49:50,547 INFO [master/828b3f090c91:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-26T21:49:50,552 INFO [master/828b3f090c91:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-26T21:49:50,552 INFO [master/828b3f090c91:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-26T21:49:50,574 INFO [master/828b3f090c91:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-26T21:49:50,582 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-26T21:49:50,583 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-26T21:49:50,585 INFO [master/828b3f090c91:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-26T21:49:50,587 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-26T21:49:50,588 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-26T21:49:50,590 INFO [master/828b3f090c91:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-26T21:49:50,593 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-26T21:49:50,594 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-26T21:49:50,596 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-26T21:49:50,597 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-26T21:49:50,614 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-26T21:49:50,615 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-26T21:49:50,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T21:49:50,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46545-0x1007e03121b0001, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T21:49:50,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T21:49:50,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T21:49:50,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:49:50,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:49:50,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:49:50,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46545-0x1007e03121b0001, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:49:50,621 INFO [master/828b3f090c91:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=828b3f090c91,44203,1732657789166, sessionid=0x1007e03121b0000, setting cluster-up flag (Was=false) 2024-11-26T21:49:50,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:49:50,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:49:50,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:49:50,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46545-0x1007e03121b0001, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:49:50,636 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-26T21:49:50,638 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=828b3f090c91,44203,1732657789166 2024-11-26T21:49:50,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:49:50,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:49:50,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46545-0x1007e03121b0001, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:49:50,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:49:50,646 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-26T21:49:50,647 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=828b3f090c91,44203,1732657789166 2024-11-26T21:49:50,653 INFO [master/828b3f090c91:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-26T21:49:50,705 INFO [RS:0;828b3f090c91:46545 {}] regionserver.HRegionServer(746): ClusterId : 6d98cb23-728a-4b03-88ab-90833a5d474e 2024-11-26T21:49:50,705 INFO [RS:2;828b3f090c91:42273 {}] regionserver.HRegionServer(746): ClusterId : 6d98cb23-728a-4b03-88ab-90833a5d474e 2024-11-26T21:49:50,705 INFO [RS:1;828b3f090c91:40359 {}] regionserver.HRegionServer(746): ClusterId : 6d98cb23-728a-4b03-88ab-90833a5d474e 2024-11-26T21:49:50,708 DEBUG [RS:1;828b3f090c91:40359 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T21:49:50,708 DEBUG [RS:0;828b3f090c91:46545 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T21:49:50,708 DEBUG [RS:2;828b3f090c91:42273 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T21:49:50,712 DEBUG [RS:0;828b3f090c91:46545 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T21:49:50,712 DEBUG [RS:2;828b3f090c91:42273 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T21:49:50,712 DEBUG [RS:1;828b3f090c91:40359 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T21:49:50,712 DEBUG [RS:0;828b3f090c91:46545 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T21:49:50,712 DEBUG [RS:1;828b3f090c91:40359 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T21:49:50,712 DEBUG [RS:2;828b3f090c91:42273 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T21:49:50,718 DEBUG [RS:2;828b3f090c91:42273 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T21:49:50,718 DEBUG [RS:0;828b3f090c91:46545 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T21:49:50,718 DEBUG [RS:1;828b3f090c91:40359 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T21:49:50,719 DEBUG [RS:0;828b3f090c91:46545 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42574ac3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=828b3f090c91/172.17.0.2:0 2024-11-26T21:49:50,719 DEBUG [RS:1;828b3f090c91:40359 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d93ffb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=828b3f090c91/172.17.0.2:0 2024-11-26T21:49:50,719 DEBUG [RS:2;828b3f090c91:42273 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594fa9b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=828b3f090c91/172.17.0.2:0 2024-11-26T21:49:50,731 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T21:49:50,733 DEBUG [RS:1;828b3f090c91:40359 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;828b3f090c91:40359 2024-11-26T21:49:50,734 DEBUG [RS:0;828b3f090c91:46545 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;828b3f090c91:46545 2024-11-26T21:49:50,736 INFO [RS:0;828b3f090c91:46545 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T21:49:50,736 INFO [RS:1;828b3f090c91:40359 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T21:49:50,736 INFO [RS:1;828b3f090c91:40359 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T21:49:50,736 INFO [RS:0;828b3f090c91:46545 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T21:49:50,736 DEBUG [RS:1;828b3f090c91:40359 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T21:49:50,736 DEBUG [RS:0;828b3f090c91:46545 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T21:49:50,737 DEBUG [RS:2;828b3f090c91:42273 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;828b3f090c91:42273 2024-11-26T21:49:50,737 INFO [RS:2;828b3f090c91:42273 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T21:49:50,737 INFO [RS:2;828b3f090c91:42273 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T21:49:50,737 DEBUG [RS:2;828b3f090c91:42273 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T21:49:50,739 INFO [RS:1;828b3f090c91:40359 {}] regionserver.HRegionServer(2659): reportForDuty to master=828b3f090c91,44203,1732657789166 with port=40359, startcode=1732657789839 2024-11-26T21:49:50,739 INFO [RS:0;828b3f090c91:46545 {}] regionserver.HRegionServer(2659): reportForDuty to master=828b3f090c91,44203,1732657789166 with port=46545, startcode=1732657789755 2024-11-26T21:49:50,740 INFO [RS:2;828b3f090c91:42273 {}] regionserver.HRegionServer(2659): reportForDuty to master=828b3f090c91,44203,1732657789166 with port=42273, startcode=1732657789877 2024-11-26T21:49:50,743 INFO [master/828b3f090c91:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-26T21:49:50,752 DEBUG [RS:2;828b3f090c91:42273 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T21:49:50,752 DEBUG [RS:1;828b3f090c91:40359 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T21:49:50,752 DEBUG [RS:0;828b3f090c91:46545 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T21:49:50,752 INFO [master/828b3f090c91:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-26T21:49:50,759 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 828b3f090c91,44203,1732657789166 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-26T21:49:50,768 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/828b3f090c91:0, corePoolSize=5, maxPoolSize=5 2024-11-26T21:49:50,768 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/828b3f090c91:0, corePoolSize=5, maxPoolSize=5 2024-11-26T21:49:50,768 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/828b3f090c91:0, corePoolSize=5, maxPoolSize=5 2024-11-26T21:49:50,768 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/828b3f090c91:0, corePoolSize=5, maxPoolSize=5 2024-11-26T21:49:50,768 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/828b3f090c91:0, corePoolSize=10, maxPoolSize=10 2024-11-26T21:49:50,768 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:49:50,769 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/828b3f090c91:0, corePoolSize=2, maxPoolSize=2 2024-11-26T21:49:50,769 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:49:50,776 INFO [master/828b3f090c91:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732657820776 2024-11-26T21:49:50,779 INFO [master/828b3f090c91:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-26T21:49:50,781 INFO [master/828b3f090c91:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-26T21:49:50,782 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T21:49:50,782 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-26T21:49:50,786 INFO [master/828b3f090c91:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-26T21:49:50,787 INFO [master/828b3f090c91:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-26T21:49:50,787 INFO [master/828b3f090c91:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-26T21:49:50,787 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54315, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T21:49:50,787 INFO [master/828b3f090c91:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-26T21:49:50,787 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41187, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T21:49:50,787 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58951, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T21:49:50,790 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:50,788 INFO [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T21:49:50,790 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-26T21:49:50,792 INFO [master/828b3f090c91:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-26T21:49:50,794 INFO [master/828b3f090c91:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-26T21:49:50,794 INFO [master/828b3f090c91:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-26T21:49:50,796 INFO [master/828b3f090c91:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-26T21:49:50,796 INFO [master/828b3f090c91:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-26T21:49:50,796 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44203 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:50,798 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/828b3f090c91:0:becomeActiveMaster-HFileCleaner.large.0-1732657790797,5,FailOnTimeoutGroup] 2024-11-26T21:49:50,800 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/828b3f090c91:0:becomeActiveMaster-HFileCleaner.small.0-1732657790798,5,FailOnTimeoutGroup] 2024-11-26T21:49:50,800 INFO [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T21:49:50,800 INFO [master/828b3f090c91:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-26T21:49:50,801 INFO [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-26T21:49:50,801 INFO [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-26T21:49:50,803 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44203 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 828b3f090c91,46545,1732657789755 2024-11-26T21:49:50,806 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44203 {}] master.ServerManager(517): Registering regionserver=828b3f090c91,46545,1732657789755 2024-11-26T21:49:50,818 DEBUG [RS:0;828b3f090c91:46545 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657 2024-11-26T21:49:50,819 DEBUG [RS:0;828b3f090c91:46545 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45749 2024-11-26T21:49:50,819 DEBUG [RS:0;828b3f090c91:46545 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T21:49:50,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T21:49:50,824 DEBUG [RS:0;828b3f090c91:46545 {}] zookeeper.ZKUtil(111): regionserver:46545-0x1007e03121b0001, quorum=127.0.0.1:53387, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/828b3f090c91,46545,1732657789755 2024-11-26T21:49:50,824 WARN [RS:0;828b3f090c91:46545 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T21:49:50,824 INFO [RS:0;828b3f090c91:46545 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T21:49:50,824 DEBUG [RS:0;828b3f090c91:46545 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,46545,1732657789755 2024-11-26T21:49:50,828 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [828b3f090c91,46545,1732657789755] 2024-11-26T21:49:50,841 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44203 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 828b3f090c91,40359,1732657789839 2024-11-26T21:49:50,841 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44203 {}] master.ServerManager(517): Registering regionserver=828b3f090c91,40359,1732657789839 2024-11-26T21:49:50,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741831_1007 (size=1321) 2024-11-26T21:49:50,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741831_1007 (size=1321) 2024-11-26T21:49:50,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741831_1007 (size=1321) 2024-11-26T21:49:50,848 DEBUG [RS:2;828b3f090c91:42273 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-26T21:49:50,848 DEBUG [RS:1;828b3f090c91:40359 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657 2024-11-26T21:49:50,848 WARN [RS:2;828b3f090c91:42273 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-26T21:49:50,848 DEBUG [RS:1;828b3f090c91:40359 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45749 2024-11-26T21:49:50,848 DEBUG [RS:1;828b3f090c91:40359 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T21:49:50,849 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-26T21:49:50,850 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657 2024-11-26T21:49:50,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T21:49:50,851 DEBUG [RS:1;828b3f090c91:40359 {}] zookeeper.ZKUtil(111): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/828b3f090c91,40359,1732657789839 2024-11-26T21:49:50,852 WARN [RS:1;828b3f090c91:40359 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T21:49:50,852 INFO [RS:1;828b3f090c91:40359 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T21:49:50,852 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [828b3f090c91,40359,1732657789839] 2024-11-26T21:49:50,852 DEBUG [RS:1;828b3f090c91:40359 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,40359,1732657789839 2024-11-26T21:49:50,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741832_1008 (size=32) 2024-11-26T21:49:50,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741832_1008 (size=32) 2024-11-26T21:49:50,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741832_1008 (size=32) 2024-11-26T21:49:50,872 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:50,875 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T21:49:50,877 INFO [RS:0;828b3f090c91:46545 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T21:49:50,877 INFO [RS:1;828b3f090c91:40359 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T21:49:50,878 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T21:49:50,878 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:50,879 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T21:49:50,880 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T21:49:50,883 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T21:49:50,883 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:50,884 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T21:49:50,885 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T21:49:50,888 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T21:49:50,888 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:50,889 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T21:49:50,889 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T21:49:50,893 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T21:49:50,893 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:50,894 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T21:49:50,894 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T21:49:50,895 INFO [RS:0;828b3f090c91:46545 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T21:49:50,895 INFO [RS:1;828b3f090c91:40359 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T21:49:50,896 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/hbase/meta/1588230740 2024-11-26T21:49:50,897 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/hbase/meta/1588230740 2024-11-26T21:49:50,900 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T21:49:50,900 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T21:49:50,901 INFO [RS:0;828b3f090c91:46545 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T21:49:50,901 INFO [RS:1;828b3f090c91:40359 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T21:49:50,901 INFO [RS:0;828b3f090c91:46545 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T21:49:50,901 INFO [RS:1;828b3f090c91:40359 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T21:49:50,902 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T21:49:50,903 INFO [RS:1;828b3f090c91:40359 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T21:49:50,903 INFO [RS:0;828b3f090c91:46545 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T21:49:50,905 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T21:49:50,910 INFO [RS:1;828b3f090c91:40359 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T21:49:50,910 INFO [RS:0;828b3f090c91:46545 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T21:49:50,912 INFO [RS:1;828b3f090c91:40359 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T21:49:50,912 INFO [RS:0;828b3f090c91:46545 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T21:49:50,912 DEBUG [RS:1;828b3f090c91:40359 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:49:50,912 DEBUG [RS:0;828b3f090c91:46545 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:49:50,913 DEBUG [RS:1;828b3f090c91:40359 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:49:50,913 DEBUG [RS:0;828b3f090c91:46545 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:49:50,913 DEBUG [RS:1;828b3f090c91:40359 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:49:50,913 DEBUG [RS:0;828b3f090c91:46545 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:49:50,913 DEBUG [RS:1;828b3f090c91:40359 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:49:50,913 DEBUG [RS:0;828b3f090c91:46545 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:49:50,913 DEBUG [RS:1;828b3f090c91:40359 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:49:50,913 DEBUG [RS:0;828b3f090c91:46545 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:49:50,913 DEBUG [RS:1;828b3f090c91:40359 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/828b3f090c91:0, corePoolSize=2, maxPoolSize=2 2024-11-26T21:49:50,913 DEBUG [RS:0;828b3f090c91:46545 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/828b3f090c91:0, corePoolSize=2, maxPoolSize=2 2024-11-26T21:49:50,913 DEBUG [RS:1;828b3f090c91:40359 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:49:50,913 DEBUG [RS:1;828b3f090c91:40359 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:49:50,914 DEBUG [RS:1;828b3f090c91:40359 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:49:50,914 DEBUG [RS:0;828b3f090c91:46545 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:49:50,914 DEBUG [RS:1;828b3f090c91:40359 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:49:50,914 DEBUG [RS:0;828b3f090c91:46545 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:49:50,914 DEBUG [RS:1;828b3f090c91:40359 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:49:50,914 DEBUG [RS:0;828b3f090c91:46545 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:49:50,914 DEBUG [RS:1;828b3f090c91:40359 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:49:50,914 DEBUG [RS:0;828b3f090c91:46545 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:49:50,914 DEBUG [RS:1;828b3f090c91:40359 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/828b3f090c91:0, corePoolSize=3, maxPoolSize=3 2024-11-26T21:49:50,914 DEBUG [RS:0;828b3f090c91:46545 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:49:50,914 DEBUG [RS:1;828b3f090c91:40359 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/828b3f090c91:0, corePoolSize=3, maxPoolSize=3 2024-11-26T21:49:50,914 DEBUG [RS:0;828b3f090c91:46545 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:49:50,914 DEBUG [RS:0;828b3f090c91:46545 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/828b3f090c91:0, corePoolSize=3, maxPoolSize=3 2024-11-26T21:49:50,914 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T21:49:50,915 DEBUG [RS:0;828b3f090c91:46545 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/828b3f090c91:0, corePoolSize=3, maxPoolSize=3 2024-11-26T21:49:50,916 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66054653, jitterRate=-0.01570896804332733}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T21:49:50,917 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732657790872Initializing all the Stores at 1732657790875 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732657790875Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732657790875Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657790875Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732657790875Cleaning up temporary data from old regions at 1732657790900 (+25 ms)Region opened successfully at 1732657790917 (+17 ms) 2024-11-26T21:49:50,917 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T21:49:50,917 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T21:49:50,917 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T21:49:50,917 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T21:49:50,917 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T21:49:50,921 INFO [RS:1;828b3f090c91:40359 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T21:49:50,921 INFO [RS:1;828b3f090c91:40359 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T21:49:50,921 INFO [RS:1;828b3f090c91:40359 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T21:49:50,921 INFO [RS:1;828b3f090c91:40359 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T21:49:50,921 INFO [RS:1;828b3f090c91:40359 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T21:49:50,922 INFO [RS:1;828b3f090c91:40359 {}] hbase.ChoreService(168): Chore ScheduledChore name=828b3f090c91,40359,1732657789839-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T21:49:50,922 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T21:49:50,922 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732657790917Disabling compacts and flushes for region at 1732657790917Disabling writes for close at 1732657790917Writing region close event to WAL at 1732657790921 (+4 ms)Closed at 1732657790922 (+1 ms) 2024-11-26T21:49:50,924 INFO [RS:0;828b3f090c91:46545 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T21:49:50,924 INFO [RS:0;828b3f090c91:46545 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T21:49:50,925 INFO [RS:0;828b3f090c91:46545 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T21:49:50,925 INFO [RS:0;828b3f090c91:46545 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T21:49:50,925 INFO [RS:0;828b3f090c91:46545 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T21:49:50,925 INFO [RS:0;828b3f090c91:46545 {}] hbase.ChoreService(168): Chore ScheduledChore name=828b3f090c91,46545,1732657789755-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T21:49:50,925 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T21:49:50,925 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-26T21:49:50,934 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-26T21:49:50,949 INFO [RS:2;828b3f090c91:42273 {}] regionserver.HRegionServer(2659): reportForDuty to master=828b3f090c91,44203,1732657789166 with port=42273, startcode=1732657789877 2024-11-26T21:49:50,951 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44203 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 828b3f090c91,42273,1732657789877 2024-11-26T21:49:50,951 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T21:49:50,951 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44203 {}] master.ServerManager(517): Registering regionserver=828b3f090c91,42273,1732657789877 2024-11-26T21:49:50,956 INFO [RS:1;828b3f090c91:40359 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T21:49:50,956 INFO [RS:0;828b3f090c91:46545 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T21:49:50,956 DEBUG [RS:2;828b3f090c91:42273 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657 2024-11-26T21:49:50,956 DEBUG [RS:2;828b3f090c91:42273 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45749 2024-11-26T21:49:50,956 DEBUG [RS:2;828b3f090c91:42273 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T21:49:50,956 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-26T21:49:50,958 INFO [RS:0;828b3f090c91:46545 {}] hbase.ChoreService(168): Chore ScheduledChore name=828b3f090c91,46545,1732657789755-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T21:49:50,958 INFO [RS:1;828b3f090c91:40359 {}] hbase.ChoreService(168): Chore ScheduledChore name=828b3f090c91,40359,1732657789839-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T21:49:50,958 INFO [RS:1;828b3f090c91:40359 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T21:49:50,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T21:49:50,958 INFO [RS:1;828b3f090c91:40359 {}] regionserver.Replication(171): 828b3f090c91,40359,1732657789839 started 2024-11-26T21:49:50,959 INFO [RS:0;828b3f090c91:46545 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T21:49:50,959 INFO [RS:0;828b3f090c91:46545 {}] regionserver.Replication(171): 828b3f090c91,46545,1732657789755 started 2024-11-26T21:49:50,960 DEBUG [RS:2;828b3f090c91:42273 {}] zookeeper.ZKUtil(111): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/828b3f090c91,42273,1732657789877 2024-11-26T21:49:50,960 WARN [RS:2;828b3f090c91:42273 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T21:49:50,960 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [828b3f090c91,42273,1732657789877] 2024-11-26T21:49:50,960 INFO [RS:2;828b3f090c91:42273 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T21:49:50,961 DEBUG [RS:2;828b3f090c91:42273 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,42273,1732657789877 2024-11-26T21:49:50,970 INFO [RS:2;828b3f090c91:42273 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T21:49:50,975 INFO [RS:2;828b3f090c91:42273 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T21:49:50,975 INFO [RS:2;828b3f090c91:42273 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T21:49:50,976 INFO [RS:2;828b3f090c91:42273 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T21:49:50,977 INFO [RS:2;828b3f090c91:42273 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T21:49:50,979 INFO [RS:2;828b3f090c91:42273 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T21:49:50,979 INFO [RS:2;828b3f090c91:42273 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T21:49:50,980 DEBUG [RS:2;828b3f090c91:42273 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:49:50,980 DEBUG [RS:2;828b3f090c91:42273 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:49:50,980 DEBUG [RS:2;828b3f090c91:42273 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:49:50,980 DEBUG [RS:2;828b3f090c91:42273 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:49:50,980 DEBUG [RS:2;828b3f090c91:42273 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:49:50,980 DEBUG [RS:2;828b3f090c91:42273 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/828b3f090c91:0, corePoolSize=2, maxPoolSize=2 2024-11-26T21:49:50,981 DEBUG [RS:2;828b3f090c91:42273 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:49:50,981 DEBUG [RS:2;828b3f090c91:42273 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:49:50,981 DEBUG [RS:2;828b3f090c91:42273 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:49:50,981 DEBUG [RS:2;828b3f090c91:42273 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:49:50,981 DEBUG [RS:2;828b3f090c91:42273 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:49:50,981 DEBUG [RS:2;828b3f090c91:42273 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:49:50,982 DEBUG [RS:2;828b3f090c91:42273 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/828b3f090c91:0, corePoolSize=3, maxPoolSize=3 2024-11-26T21:49:50,982 DEBUG [RS:2;828b3f090c91:42273 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/828b3f090c91:0, corePoolSize=3, maxPoolSize=3 2024-11-26T21:49:50,982 INFO [RS:2;828b3f090c91:42273 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T21:49:50,982 INFO [RS:2;828b3f090c91:42273 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T21:49:50,983 INFO [RS:2;828b3f090c91:42273 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T21:49:50,983 INFO [RS:2;828b3f090c91:42273 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T21:49:50,983 INFO [RS:2;828b3f090c91:42273 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T21:49:50,983 INFO [RS:2;828b3f090c91:42273 {}] hbase.ChoreService(168): Chore ScheduledChore name=828b3f090c91,42273,1732657789877-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T21:49:50,984 INFO [RS:1;828b3f090c91:40359 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T21:49:50,984 INFO [RS:0;828b3f090c91:46545 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T21:49:50,985 INFO [RS:0;828b3f090c91:46545 {}] regionserver.HRegionServer(1482): Serving as 828b3f090c91,46545,1732657789755, RpcServer on 828b3f090c91/172.17.0.2:46545, sessionid=0x1007e03121b0001 2024-11-26T21:49:50,985 INFO [RS:1;828b3f090c91:40359 {}] regionserver.HRegionServer(1482): Serving as 828b3f090c91,40359,1732657789839, RpcServer on 828b3f090c91/172.17.0.2:40359, sessionid=0x1007e03121b0002 2024-11-26T21:49:50,986 DEBUG [RS:0;828b3f090c91:46545 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T21:49:50,986 DEBUG [RS:1;828b3f090c91:40359 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T21:49:50,986 DEBUG [RS:0;828b3f090c91:46545 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 828b3f090c91,46545,1732657789755 2024-11-26T21:49:50,986 DEBUG [RS:1;828b3f090c91:40359 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 828b3f090c91,40359,1732657789839 2024-11-26T21:49:50,986 DEBUG [RS:1;828b3f090c91:40359 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '828b3f090c91,40359,1732657789839' 2024-11-26T21:49:50,986 DEBUG [RS:0;828b3f090c91:46545 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '828b3f090c91,46545,1732657789755' 2024-11-26T21:49:50,986 DEBUG [RS:1;828b3f090c91:40359 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T21:49:50,986 DEBUG [RS:0;828b3f090c91:46545 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T21:49:50,987 DEBUG [RS:0;828b3f090c91:46545 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T21:49:50,987 DEBUG [RS:1;828b3f090c91:40359 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T21:49:50,988 DEBUG [RS:1;828b3f090c91:40359 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T21:49:50,988 DEBUG [RS:0;828b3f090c91:46545 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T21:49:50,988 DEBUG [RS:0;828b3f090c91:46545 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T21:49:50,988 DEBUG [RS:1;828b3f090c91:40359 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T21:49:50,988 DEBUG [RS:1;828b3f090c91:40359 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 828b3f090c91,40359,1732657789839 2024-11-26T21:49:50,988 DEBUG [RS:0;828b3f090c91:46545 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 828b3f090c91,46545,1732657789755 2024-11-26T21:49:50,989 DEBUG [RS:1;828b3f090c91:40359 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '828b3f090c91,40359,1732657789839' 2024-11-26T21:49:50,989 DEBUG [RS:0;828b3f090c91:46545 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '828b3f090c91,46545,1732657789755' 2024-11-26T21:49:50,989 DEBUG [RS:1;828b3f090c91:40359 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T21:49:50,989 DEBUG [RS:0;828b3f090c91:46545 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T21:49:50,989 DEBUG [RS:1;828b3f090c91:40359 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T21:49:50,989 DEBUG [RS:0;828b3f090c91:46545 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T21:49:50,990 DEBUG [RS:0;828b3f090c91:46545 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T21:49:50,990 DEBUG [RS:1;828b3f090c91:40359 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T21:49:50,990 INFO [RS:0;828b3f090c91:46545 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T21:49:50,990 INFO [RS:1;828b3f090c91:40359 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T21:49:50,990 INFO [RS:0;828b3f090c91:46545 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T21:49:50,990 INFO [RS:1;828b3f090c91:40359 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T21:49:51,006 INFO [RS:2;828b3f090c91:42273 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T21:49:51,007 INFO [RS:2;828b3f090c91:42273 {}] hbase.ChoreService(168): Chore ScheduledChore name=828b3f090c91,42273,1732657789877-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T21:49:51,007 INFO [RS:2;828b3f090c91:42273 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T21:49:51,007 INFO [RS:2;828b3f090c91:42273 {}] regionserver.Replication(171): 828b3f090c91,42273,1732657789877 started 2024-11-26T21:49:51,029 INFO [RS:2;828b3f090c91:42273 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T21:49:51,029 INFO [RS:2;828b3f090c91:42273 {}] regionserver.HRegionServer(1482): Serving as 828b3f090c91,42273,1732657789877, RpcServer on 828b3f090c91/172.17.0.2:42273, sessionid=0x1007e03121b0003 2024-11-26T21:49:51,030 DEBUG [RS:2;828b3f090c91:42273 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T21:49:51,030 DEBUG [RS:2;828b3f090c91:42273 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 828b3f090c91,42273,1732657789877 2024-11-26T21:49:51,030 DEBUG [RS:2;828b3f090c91:42273 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '828b3f090c91,42273,1732657789877' 2024-11-26T21:49:51,030 DEBUG [RS:2;828b3f090c91:42273 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T21:49:51,031 DEBUG [RS:2;828b3f090c91:42273 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T21:49:51,031 DEBUG [RS:2;828b3f090c91:42273 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T21:49:51,032 DEBUG [RS:2;828b3f090c91:42273 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T21:49:51,032 DEBUG [RS:2;828b3f090c91:42273 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 828b3f090c91,42273,1732657789877 2024-11-26T21:49:51,032 DEBUG [RS:2;828b3f090c91:42273 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '828b3f090c91,42273,1732657789877' 2024-11-26T21:49:51,032 DEBUG [RS:2;828b3f090c91:42273 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T21:49:51,032 DEBUG [RS:2;828b3f090c91:42273 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T21:49:51,033 DEBUG [RS:2;828b3f090c91:42273 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T21:49:51,033 INFO [RS:2;828b3f090c91:42273 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T21:49:51,033 INFO [RS:2;828b3f090c91:42273 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T21:49:51,095 INFO [RS:1;828b3f090c91:40359 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T21:49:51,095 INFO [RS:0;828b3f090c91:46545 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T21:49:51,099 INFO [RS:0;828b3f090c91:46545 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=828b3f090c91%2C46545%2C1732657789755, suffix=, logDir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,46545,1732657789755, archiveDir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/oldWALs, maxLogs=32 2024-11-26T21:49:51,099 INFO [RS:1;828b3f090c91:40359 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=828b3f090c91%2C40359%2C1732657789839, suffix=, logDir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,40359,1732657789839, archiveDir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/oldWALs, maxLogs=32 2024-11-26T21:49:51,107 WARN [828b3f090c91:44203 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-26T21:49:51,115 DEBUG [RS:0;828b3f090c91:46545 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,46545,1732657789755/828b3f090c91%2C46545%2C1732657789755.1732657791102, exclude list is [], retry=0 2024-11-26T21:49:51,117 DEBUG [RS:1;828b3f090c91:40359 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,40359,1732657789839/828b3f090c91%2C40359%2C1732657789839.1732657791102, exclude list is [], retry=0 2024-11-26T21:49:51,121 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40345,DS-4fbda298-e9f3-436b-9808-ab5ec6eece62,DISK] 2024-11-26T21:49:51,121 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40345,DS-4fbda298-e9f3-436b-9808-ab5ec6eece62,DISK] 2024-11-26T21:49:51,121 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43659,DS-7dc835d1-8731-44e2-b974-f85d5c6f47f1,DISK] 2024-11-26T21:49:51,122 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39537,DS-aed5038f-ce60-4629-a808-1e2f6bde72d8,DISK] 2024-11-26T21:49:51,122 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43659,DS-7dc835d1-8731-44e2-b974-f85d5c6f47f1,DISK] 2024-11-26T21:49:51,122 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39537,DS-aed5038f-ce60-4629-a808-1e2f6bde72d8,DISK] 2024-11-26T21:49:51,131 INFO [RS:0;828b3f090c91:46545 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,46545,1732657789755/828b3f090c91%2C46545%2C1732657789755.1732657791102 2024-11-26T21:49:51,133 DEBUG [RS:0;828b3f090c91:46545 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36781:36781),(127.0.0.1/127.0.0.1:45705:45705),(127.0.0.1/127.0.0.1:36805:36805)] 2024-11-26T21:49:51,134 INFO [RS:2;828b3f090c91:42273 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T21:49:51,135 INFO [RS:1;828b3f090c91:40359 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,40359,1732657789839/828b3f090c91%2C40359%2C1732657789839.1732657791102 2024-11-26T21:49:51,136 DEBUG [RS:1;828b3f090c91:40359 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45705:45705),(127.0.0.1/127.0.0.1:36781:36781),(127.0.0.1/127.0.0.1:36805:36805)] 2024-11-26T21:49:51,137 INFO [RS:2;828b3f090c91:42273 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=828b3f090c91%2C42273%2C1732657789877, suffix=, logDir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,42273,1732657789877, archiveDir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/oldWALs, maxLogs=32 2024-11-26T21:49:51,154 DEBUG [RS:2;828b3f090c91:42273 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,42273,1732657789877/828b3f090c91%2C42273%2C1732657789877.1732657791139, exclude list is [], retry=0 2024-11-26T21:49:51,158 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43659,DS-7dc835d1-8731-44e2-b974-f85d5c6f47f1,DISK] 2024-11-26T21:49:51,158 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40345,DS-4fbda298-e9f3-436b-9808-ab5ec6eece62,DISK] 2024-11-26T21:49:51,159 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39537,DS-aed5038f-ce60-4629-a808-1e2f6bde72d8,DISK] 2024-11-26T21:49:51,163 INFO [RS:2;828b3f090c91:42273 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,42273,1732657789877/828b3f090c91%2C42273%2C1732657789877.1732657791139 2024-11-26T21:49:51,163 DEBUG [RS:2;828b3f090c91:42273 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36781:36781),(127.0.0.1/127.0.0.1:36805:36805),(127.0.0.1/127.0.0.1:45705:45705)] 2024-11-26T21:49:51,364 DEBUG [828b3f090c91:44203 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-26T21:49:51,372 DEBUG [828b3f090c91:44203 {}] balancer.BalancerClusterState(204): Hosts are {828b3f090c91=0} racks are {/default-rack=0} 2024-11-26T21:49:51,378 DEBUG [828b3f090c91:44203 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T21:49:51,378 DEBUG [828b3f090c91:44203 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T21:49:51,378 DEBUG [828b3f090c91:44203 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-26T21:49:51,378 DEBUG [828b3f090c91:44203 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T21:49:51,378 DEBUG [828b3f090c91:44203 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T21:49:51,378 DEBUG [828b3f090c91:44203 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-26T21:49:51,378 INFO [828b3f090c91:44203 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T21:49:51,378 INFO [828b3f090c91:44203 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T21:49:51,378 INFO [828b3f090c91:44203 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-26T21:49:51,378 DEBUG [828b3f090c91:44203 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T21:49:51,385 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=828b3f090c91,46545,1732657789755 2024-11-26T21:49:51,391 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 828b3f090c91,46545,1732657789755, state=OPENING 2024-11-26T21:49:51,396 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-26T21:49:51,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:49:51,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:49:51,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:49:51,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46545-0x1007e03121b0001, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:49:51,398 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:51,398 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:51,398 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:51,398 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:51,400 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T21:49:51,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,46545,1732657789755}] 2024-11-26T21:49:51,584 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T21:49:51,587 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55731, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T21:49:51,598 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-26T21:49:51,598 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T21:49:51,599 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-26T21:49:51,602 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=828b3f090c91%2C46545%2C1732657789755.meta, suffix=.meta, logDir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,46545,1732657789755, archiveDir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/oldWALs, maxLogs=32 2024-11-26T21:49:51,618 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,46545,1732657789755/828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, exclude list is [], retry=0 2024-11-26T21:49:51,622 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39537,DS-aed5038f-ce60-4629-a808-1e2f6bde72d8,DISK] 2024-11-26T21:49:51,623 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43659,DS-7dc835d1-8731-44e2-b974-f85d5c6f47f1,DISK] 2024-11-26T21:49:51,623 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40345,DS-4fbda298-e9f3-436b-9808-ab5ec6eece62,DISK] 2024-11-26T21:49:51,625 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,46545,1732657789755/828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta 2024-11-26T21:49:51,627 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45705:45705),(127.0.0.1/127.0.0.1:36781:36781),(127.0.0.1/127.0.0.1:36805:36805)] 2024-11-26T21:49:51,627 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-26T21:49:51,629 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-26T21:49:51,631 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-26T21:49:51,636 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-26T21:49:51,640 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-26T21:49:51,641 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:51,641 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-26T21:49:51,641 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-26T21:49:51,644 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T21:49:51,646 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T21:49:51,646 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:51,647 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T21:49:51,647 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T21:49:51,648 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T21:49:51,649 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:51,649 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T21:49:51,650 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T21:49:51,651 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T21:49:51,651 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:51,652 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T21:49:51,652 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T21:49:51,653 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T21:49:51,653 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:51,654 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T21:49:51,654 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T21:49:51,656 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/hbase/meta/1588230740 2024-11-26T21:49:51,659 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/hbase/meta/1588230740 2024-11-26T21:49:51,662 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T21:49:51,662 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T21:49:51,663 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T21:49:51,667 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T21:49:51,668 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70707939, jitterRate=0.05363039672374725}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T21:49:51,668 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-26T21:49:51,670 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732657791641Writing region info on filesystem at 1732657791642 (+1 ms)Initializing all the Stores at 1732657791644 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732657791644Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732657791644Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657791644Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732657791644Cleaning up temporary data from old regions at 1732657791662 (+18 ms)Running coprocessor post-open hooks at 1732657791669 (+7 ms)Region opened successfully at 1732657791670 (+1 ms) 2024-11-26T21:49:51,677 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732657791572 2024-11-26T21:49:51,688 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-26T21:49:51,689 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-26T21:49:51,690 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=828b3f090c91,46545,1732657789755 2024-11-26T21:49:51,692 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 828b3f090c91,46545,1732657789755, state=OPEN 2024-11-26T21:49:51,694 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:51,694 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:51,694 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46545-0x1007e03121b0001, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:51,694 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:51,694 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:51,694 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:51,694 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:51,694 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:51,695 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=828b3f090c91,46545,1732657789755 2024-11-26T21:49:51,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-26T21:49:51,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,46545,1732657789755 in 293 msec 2024-11-26T21:49:51,707 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-26T21:49:51,707 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 769 msec 2024-11-26T21:49:51,708 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T21:49:51,708 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-26T21:49:51,727 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T21:49:51,728 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=828b3f090c91,46545,1732657789755, seqNum=-1] 2024-11-26T21:49:51,746 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T21:49:51,748 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46589, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T21:49:51,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0820 sec 2024-11-26T21:49:51,766 INFO [master/828b3f090c91:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732657791766, completionTime=-1 2024-11-26T21:49:51,769 INFO [master/828b3f090c91:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-26T21:49:51,769 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T21:49:51,797 INFO [master/828b3f090c91:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-26T21:49:51,797 INFO [master/828b3f090c91:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732657851797 2024-11-26T21:49:51,797 INFO [master/828b3f090c91:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732657911797 2024-11-26T21:49:51,797 INFO [master/828b3f090c91:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 28 msec 2024-11-26T21:49:51,799 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-26T21:49:51,805 INFO [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=828b3f090c91,44203,1732657789166-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T21:49:51,806 INFO [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=828b3f090c91,44203,1732657789166-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T21:49:51,806 INFO [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=828b3f090c91,44203,1732657789166-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T21:49:51,807 INFO [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-828b3f090c91:44203, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T21:49:51,808 INFO [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-26T21:49:51,808 INFO [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-26T21:49:51,814 DEBUG [master/828b3f090c91:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-26T21:49:51,836 INFO [master/828b3f090c91:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.875sec 2024-11-26T21:49:51,838 INFO [master/828b3f090c91:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-26T21:49:51,839 INFO [master/828b3f090c91:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-26T21:49:51,840 INFO [master/828b3f090c91:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-26T21:49:51,841 INFO [master/828b3f090c91:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-26T21:49:51,841 INFO [master/828b3f090c91:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-26T21:49:51,841 INFO [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=828b3f090c91,44203,1732657789166-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T21:49:51,842 INFO [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=828b3f090c91,44203,1732657789166-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-26T21:49:51,846 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-26T21:49:51,847 INFO [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-26T21:49:51,848 INFO [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=828b3f090c91,44203,1732657789166-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T21:49:51,915 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@359847dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T21:49:51,920 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-26T21:49:51,920 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-26T21:49:51,924 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 828b3f090c91,44203,-1 for getting cluster id 2024-11-26T21:49:51,943 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T21:49:51,955 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6d98cb23-728a-4b03-88ab-90833a5d474e' 2024-11-26T21:49:51,958 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T21:49:51,959 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6d98cb23-728a-4b03-88ab-90833a5d474e" 2024-11-26T21:49:51,959 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@566db3d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T21:49:51,959 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [828b3f090c91,44203,-1] 2024-11-26T21:49:51,962 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T21:49:51,964 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T21:49:51,966 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60306, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T21:49:51,973 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b221f9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T21:49:51,974 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T21:49:51,982 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=828b3f090c91,46545,1732657789755, seqNum=-1] 2024-11-26T21:49:51,983 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T21:49:51,986 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40606, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T21:49:52,007 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=828b3f090c91,44203,1732657789166 2024-11-26T21:49:52,011 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T21:49:52,016 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 828b3f090c91,44203,1732657789166 2024-11-26T21:49:52,018 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@602ac356 2024-11-26T21:49:52,019 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T21:49:52,021 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60318, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T21:49:52,029 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T21:49:52,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-11-26T21:49:52,041 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T21:49:52,043 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-11-26T21:49:52,044 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:52,047 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T21:49:52,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T21:49:52,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741837_1013 (size=358) 2024-11-26T21:49:52,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741837_1013 (size=358) 2024-11-26T21:49:52,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741837_1013 (size=358) 2024-11-26T21:49:52,065 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => bf200defbfcf83538e813f9fa91d9c35, NAME => 'test,,1732657792023.bf200defbfcf83538e813f9fa91d9c35.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657 2024-11-26T21:49:52,065 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 34e003c01eb3f0a2e10e33152402164c, NAME => 'test,ccc,1732657792023.34e003c01eb3f0a2e10e33152402164c.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657 2024-11-26T21:49:52,066 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 36699c847bc0e6c6b06fd8533d3c9edb, NAME => 'test,bbb,1732657792023.36699c847bc0e6c6b06fd8533d3c9edb.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657 2024-11-26T21:49:52,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741838_1014 (size=45) 2024-11-26T21:49:52,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741839_1015 (size=45) 2024-11-26T21:49:52,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741839_1015 (size=45) 2024-11-26T21:49:52,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741839_1015 (size=45) 2024-11-26T21:49:52,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741838_1014 (size=45) 2024-11-26T21:49:52,090 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1732657792023.36699c847bc0e6c6b06fd8533d3c9edb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:52,090 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 36699c847bc0e6c6b06fd8533d3c9edb, disabling compactions & flushes 2024-11-26T21:49:52,090 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1732657792023.36699c847bc0e6c6b06fd8533d3c9edb. 2024-11-26T21:49:52,090 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1732657792023.36699c847bc0e6c6b06fd8533d3c9edb. 2024-11-26T21:49:52,090 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1732657792023.36699c847bc0e6c6b06fd8533d3c9edb. after waiting 0 ms 2024-11-26T21:49:52,090 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1732657792023.36699c847bc0e6c6b06fd8533d3c9edb. 2024-11-26T21:49:52,090 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1732657792023.36699c847bc0e6c6b06fd8533d3c9edb. 2024-11-26T21:49:52,091 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 36699c847bc0e6c6b06fd8533d3c9edb: Waiting for close lock at 1732657792090Disabling compacts and flushes for region at 1732657792090Disabling writes for close at 1732657792090Writing region close event to WAL at 1732657792090Closed at 1732657792090 2024-11-26T21:49:52,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741838_1014 (size=45) 2024-11-26T21:49:52,091 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 7a7c374815f888db179beadaf2489ad7, NAME => 'test,ddd,1732657792023.7a7c374815f888db179beadaf2489ad7.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657 2024-11-26T21:49:52,091 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1732657792023.34e003c01eb3f0a2e10e33152402164c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:52,091 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 34e003c01eb3f0a2e10e33152402164c, disabling compactions & flushes 2024-11-26T21:49:52,092 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1732657792023.34e003c01eb3f0a2e10e33152402164c. 2024-11-26T21:49:52,092 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1732657792023.34e003c01eb3f0a2e10e33152402164c. 2024-11-26T21:49:52,092 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1732657792023.34e003c01eb3f0a2e10e33152402164c. after waiting 0 ms 2024-11-26T21:49:52,092 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1732657792023.34e003c01eb3f0a2e10e33152402164c. 2024-11-26T21:49:52,092 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1732657792023.34e003c01eb3f0a2e10e33152402164c. 2024-11-26T21:49:52,092 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 34e003c01eb3f0a2e10e33152402164c: Waiting for close lock at 1732657792091Disabling compacts and flushes for region at 1732657792091Disabling writes for close at 1732657792092 (+1 ms)Writing region close event to WAL at 1732657792092Closed at 1732657792092 2024-11-26T21:49:52,093 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 651cf53d9624e6cc98e46d6578c71496, NAME => 'test,eee,1732657792023.651cf53d9624e6cc98e46d6578c71496.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657 2024-11-26T21:49:52,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741840_1016 (size=42) 2024-11-26T21:49:52,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741840_1016 (size=42) 2024-11-26T21:49:52,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741840_1016 (size=42) 2024-11-26T21:49:52,097 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1732657792023.bf200defbfcf83538e813f9fa91d9c35.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:52,097 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing bf200defbfcf83538e813f9fa91d9c35, disabling compactions & flushes 2024-11-26T21:49:52,097 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1732657792023.bf200defbfcf83538e813f9fa91d9c35. 2024-11-26T21:49:52,097 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1732657792023.bf200defbfcf83538e813f9fa91d9c35. 2024-11-26T21:49:52,097 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1732657792023.bf200defbfcf83538e813f9fa91d9c35. after waiting 0 ms 2024-11-26T21:49:52,097 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1732657792023.bf200defbfcf83538e813f9fa91d9c35. 2024-11-26T21:49:52,097 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1732657792023.bf200defbfcf83538e813f9fa91d9c35. 2024-11-26T21:49:52,097 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for bf200defbfcf83538e813f9fa91d9c35: Waiting for close lock at 1732657792097Disabling compacts and flushes for region at 1732657792097Disabling writes for close at 1732657792097Writing region close event to WAL at 1732657792097Closed at 1732657792097 2024-11-26T21:49:52,098 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f0a1ab633a90e6b5f5e9ba8ce6f66010, NAME => 'test,fff,1732657792023.f0a1ab633a90e6b5f5e9ba8ce6f66010.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657 2024-11-26T21:49:52,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741842_1018 (size=45) 2024-11-26T21:49:52,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741842_1018 (size=45) 2024-11-26T21:49:52,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741841_1017 (size=45) 2024-11-26T21:49:52,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741841_1017 (size=45) 2024-11-26T21:49:52,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741843_1019 (size=45) 2024-11-26T21:49:52,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741842_1018 (size=45) 2024-11-26T21:49:52,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741843_1019 (size=45) 2024-11-26T21:49:52,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741841_1017 (size=45) 2024-11-26T21:49:52,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741843_1019 (size=45) 2024-11-26T21:49:52,120 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,fff,1732657792023.f0a1ab633a90e6b5f5e9ba8ce6f66010.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:52,120 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing f0a1ab633a90e6b5f5e9ba8ce6f66010, disabling compactions & flushes 2024-11-26T21:49:52,120 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,fff,1732657792023.f0a1ab633a90e6b5f5e9ba8ce6f66010. 2024-11-26T21:49:52,120 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1732657792023.f0a1ab633a90e6b5f5e9ba8ce6f66010. 2024-11-26T21:49:52,120 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1732657792023.f0a1ab633a90e6b5f5e9ba8ce6f66010. after waiting 0 ms 2024-11-26T21:49:52,120 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1732657792023.f0a1ab633a90e6b5f5e9ba8ce6f66010. 2024-11-26T21:49:52,120 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,fff,1732657792023.f0a1ab633a90e6b5f5e9ba8ce6f66010. 2024-11-26T21:49:52,120 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for f0a1ab633a90e6b5f5e9ba8ce6f66010: Waiting for close lock at 1732657792120Disabling compacts and flushes for region at 1732657792120Disabling writes for close at 1732657792120Writing region close event to WAL at 1732657792120Closed at 1732657792120 2024-11-26T21:49:52,121 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8be22ac0f4cc7a17874b9d10ba204518, NAME => 'test,ggg,1732657792023.8be22ac0f4cc7a17874b9d10ba204518.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657 2024-11-26T21:49:52,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741844_1020 (size=45) 2024-11-26T21:49:52,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741844_1020 (size=45) 2024-11-26T21:49:52,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741844_1020 (size=45) 2024-11-26T21:49:52,136 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ggg,1732657792023.8be22ac0f4cc7a17874b9d10ba204518.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:52,136 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 8be22ac0f4cc7a17874b9d10ba204518, disabling compactions & flushes 2024-11-26T21:49:52,136 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ggg,1732657792023.8be22ac0f4cc7a17874b9d10ba204518. 2024-11-26T21:49:52,136 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732657792023.8be22ac0f4cc7a17874b9d10ba204518. 2024-11-26T21:49:52,136 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732657792023.8be22ac0f4cc7a17874b9d10ba204518. after waiting 0 ms 2024-11-26T21:49:52,136 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732657792023.8be22ac0f4cc7a17874b9d10ba204518. 2024-11-26T21:49:52,136 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ggg,1732657792023.8be22ac0f4cc7a17874b9d10ba204518. 2024-11-26T21:49:52,136 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8be22ac0f4cc7a17874b9d10ba204518: Waiting for close lock at 1732657792136Disabling compacts and flushes for region at 1732657792136Disabling writes for close at 1732657792136Writing region close event to WAL at 1732657792136Closed at 1732657792136 2024-11-26T21:49:52,137 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a2d6b8d1f56582e3f52e3cfe7a116c1d, NAME => 'test,hhh,1732657792023.a2d6b8d1f56582e3f52e3cfe7a116c1d.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657 2024-11-26T21:49:52,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741845_1021 (size=45) 2024-11-26T21:49:52,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741845_1021 (size=45) 2024-11-26T21:49:52,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741845_1021 (size=45) 2024-11-26T21:49:52,148 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,hhh,1732657792023.a2d6b8d1f56582e3f52e3cfe7a116c1d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:52,149 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing a2d6b8d1f56582e3f52e3cfe7a116c1d, disabling compactions & flushes 2024-11-26T21:49:52,149 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,hhh,1732657792023.a2d6b8d1f56582e3f52e3cfe7a116c1d. 2024-11-26T21:49:52,149 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1732657792023.a2d6b8d1f56582e3f52e3cfe7a116c1d. 2024-11-26T21:49:52,149 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1732657792023.a2d6b8d1f56582e3f52e3cfe7a116c1d. after waiting 0 ms 2024-11-26T21:49:52,149 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1732657792023.a2d6b8d1f56582e3f52e3cfe7a116c1d. 2024-11-26T21:49:52,149 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,hhh,1732657792023.a2d6b8d1f56582e3f52e3cfe7a116c1d. 2024-11-26T21:49:52,149 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for a2d6b8d1f56582e3f52e3cfe7a116c1d: Waiting for close lock at 1732657792149Disabling compacts and flushes for region at 1732657792149Disabling writes for close at 1732657792149Writing region close event to WAL at 1732657792149Closed at 1732657792149 2024-11-26T21:49:52,149 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d22f8ae31d7e7a4d0c284e3b9676a393, NAME => 'test,iii,1732657792023.d22f8ae31d7e7a4d0c284e3b9676a393.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657 2024-11-26T21:49:52,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T21:49:52,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741846_1022 (size=45) 2024-11-26T21:49:52,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741846_1022 (size=45) 2024-11-26T21:49:52,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741846_1022 (size=45) 2024-11-26T21:49:52,161 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,iii,1732657792023.d22f8ae31d7e7a4d0c284e3b9676a393.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:52,162 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing d22f8ae31d7e7a4d0c284e3b9676a393, disabling compactions & flushes 2024-11-26T21:49:52,162 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,iii,1732657792023.d22f8ae31d7e7a4d0c284e3b9676a393. 2024-11-26T21:49:52,162 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1732657792023.d22f8ae31d7e7a4d0c284e3b9676a393. 2024-11-26T21:49:52,162 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1732657792023.d22f8ae31d7e7a4d0c284e3b9676a393. after waiting 0 ms 2024-11-26T21:49:52,162 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1732657792023.d22f8ae31d7e7a4d0c284e3b9676a393. 2024-11-26T21:49:52,162 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,iii,1732657792023.d22f8ae31d7e7a4d0c284e3b9676a393. 2024-11-26T21:49:52,162 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for d22f8ae31d7e7a4d0c284e3b9676a393: Waiting for close lock at 1732657792161Disabling compacts and flushes for region at 1732657792161Disabling writes for close at 1732657792162 (+1 ms)Writing region close event to WAL at 1732657792162Closed at 1732657792162 2024-11-26T21:49:52,162 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cdd2d12bbd55678c7f638b4ef0fde3da, NAME => 'test,jjj,1732657792023.cdd2d12bbd55678c7f638b4ef0fde3da.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657 2024-11-26T21:49:52,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741847_1023 (size=45) 2024-11-26T21:49:52,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741847_1023 (size=45) 2024-11-26T21:49:52,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741847_1023 (size=45) 2024-11-26T21:49:52,176 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,jjj,1732657792023.cdd2d12bbd55678c7f638b4ef0fde3da.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:52,176 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing cdd2d12bbd55678c7f638b4ef0fde3da, disabling compactions & flushes 2024-11-26T21:49:52,176 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,jjj,1732657792023.cdd2d12bbd55678c7f638b4ef0fde3da. 2024-11-26T21:49:52,176 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732657792023.cdd2d12bbd55678c7f638b4ef0fde3da. 2024-11-26T21:49:52,176 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732657792023.cdd2d12bbd55678c7f638b4ef0fde3da. after waiting 0 ms 2024-11-26T21:49:52,176 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732657792023.cdd2d12bbd55678c7f638b4ef0fde3da. 2024-11-26T21:49:52,176 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,jjj,1732657792023.cdd2d12bbd55678c7f638b4ef0fde3da. 2024-11-26T21:49:52,176 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for cdd2d12bbd55678c7f638b4ef0fde3da: Waiting for close lock at 1732657792176Disabling compacts and flushes for region at 1732657792176Disabling writes for close at 1732657792176Writing region close event to WAL at 1732657792176Closed at 1732657792176 2024-11-26T21:49:52,177 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 334949367d92cc18a393737b69358f37, NAME => 'test,kkk,1732657792023.334949367d92cc18a393737b69358f37.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657 2024-11-26T21:49:52,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741848_1024 (size=45) 2024-11-26T21:49:52,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741848_1024 (size=45) 2024-11-26T21:49:52,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741848_1024 (size=45) 2024-11-26T21:49:52,188 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,kkk,1732657792023.334949367d92cc18a393737b69358f37.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:52,189 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 334949367d92cc18a393737b69358f37, disabling compactions & flushes 2024-11-26T21:49:52,189 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,kkk,1732657792023.334949367d92cc18a393737b69358f37. 2024-11-26T21:49:52,189 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1732657792023.334949367d92cc18a393737b69358f37. 2024-11-26T21:49:52,189 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1732657792023.334949367d92cc18a393737b69358f37. after waiting 0 ms 2024-11-26T21:49:52,189 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1732657792023.334949367d92cc18a393737b69358f37. 2024-11-26T21:49:52,189 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,kkk,1732657792023.334949367d92cc18a393737b69358f37. 2024-11-26T21:49:52,189 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 334949367d92cc18a393737b69358f37: Waiting for close lock at 1732657792189Disabling compacts and flushes for region at 1732657792189Disabling writes for close at 1732657792189Writing region close event to WAL at 1732657792189Closed at 1732657792189 2024-11-26T21:49:52,190 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ac402503c294574c0e555c45a12f0a4e, NAME => 'test,lll,1732657792023.ac402503c294574c0e555c45a12f0a4e.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657 2024-11-26T21:49:52,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741849_1025 (size=45) 2024-11-26T21:49:52,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741849_1025 (size=45) 2024-11-26T21:49:52,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741849_1025 (size=45) 2024-11-26T21:49:52,203 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,lll,1732657792023.ac402503c294574c0e555c45a12f0a4e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:52,203 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing ac402503c294574c0e555c45a12f0a4e, disabling compactions & flushes 2024-11-26T21:49:52,203 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,lll,1732657792023.ac402503c294574c0e555c45a12f0a4e. 2024-11-26T21:49:52,203 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1732657792023.ac402503c294574c0e555c45a12f0a4e. 2024-11-26T21:49:52,203 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1732657792023.ac402503c294574c0e555c45a12f0a4e. after waiting 0 ms 2024-11-26T21:49:52,203 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1732657792023.ac402503c294574c0e555c45a12f0a4e. 2024-11-26T21:49:52,203 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,lll,1732657792023.ac402503c294574c0e555c45a12f0a4e. 2024-11-26T21:49:52,203 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for ac402503c294574c0e555c45a12f0a4e: Waiting for close lock at 1732657792203Disabling compacts and flushes for region at 1732657792203Disabling writes for close at 1732657792203Writing region close event to WAL at 1732657792203Closed at 1732657792203 2024-11-26T21:49:52,204 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 06a1142ff2ac98bd39e6f73547d6aa98, NAME => 'test,mmm,1732657792023.06a1142ff2ac98bd39e6f73547d6aa98.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657 2024-11-26T21:49:52,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741850_1026 (size=45) 2024-11-26T21:49:52,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741850_1026 (size=45) 2024-11-26T21:49:52,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741850_1026 (size=45) 2024-11-26T21:49:52,215 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,mmm,1732657792023.06a1142ff2ac98bd39e6f73547d6aa98.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:52,215 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 06a1142ff2ac98bd39e6f73547d6aa98, disabling compactions & flushes 2024-11-26T21:49:52,215 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,mmm,1732657792023.06a1142ff2ac98bd39e6f73547d6aa98. 2024-11-26T21:49:52,215 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1732657792023.06a1142ff2ac98bd39e6f73547d6aa98. 2024-11-26T21:49:52,215 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1732657792023.06a1142ff2ac98bd39e6f73547d6aa98. after waiting 0 ms 2024-11-26T21:49:52,215 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1732657792023.06a1142ff2ac98bd39e6f73547d6aa98. 2024-11-26T21:49:52,215 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,mmm,1732657792023.06a1142ff2ac98bd39e6f73547d6aa98. 2024-11-26T21:49:52,216 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 06a1142ff2ac98bd39e6f73547d6aa98: Waiting for close lock at 1732657792215Disabling compacts and flushes for region at 1732657792215Disabling writes for close at 1732657792215Writing region close event to WAL at 1732657792215Closed at 1732657792215 2024-11-26T21:49:52,216 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c4145a9c4106377623484a3eb786fe8a, NAME => 'test,nnn,1732657792023.c4145a9c4106377623484a3eb786fe8a.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657 2024-11-26T21:49:52,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741851_1027 (size=45) 2024-11-26T21:49:52,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741851_1027 (size=45) 2024-11-26T21:49:52,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741851_1027 (size=45) 2024-11-26T21:49:52,232 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,nnn,1732657792023.c4145a9c4106377623484a3eb786fe8a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:52,232 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing c4145a9c4106377623484a3eb786fe8a, disabling compactions & flushes 2024-11-26T21:49:52,232 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,nnn,1732657792023.c4145a9c4106377623484a3eb786fe8a. 2024-11-26T21:49:52,232 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1732657792023.c4145a9c4106377623484a3eb786fe8a. 2024-11-26T21:49:52,233 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1732657792023.c4145a9c4106377623484a3eb786fe8a. after waiting 0 ms 2024-11-26T21:49:52,233 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1732657792023.c4145a9c4106377623484a3eb786fe8a. 2024-11-26T21:49:52,233 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,nnn,1732657792023.c4145a9c4106377623484a3eb786fe8a. 2024-11-26T21:49:52,233 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for c4145a9c4106377623484a3eb786fe8a: Waiting for close lock at 1732657792232Disabling compacts and flushes for region at 1732657792232Disabling writes for close at 1732657792233 (+1 ms)Writing region close event to WAL at 1732657792233Closed at 1732657792233 2024-11-26T21:49:52,234 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8a6bb8489f79432157c419605ac84965, NAME => 'test,ooo,1732657792023.8a6bb8489f79432157c419605ac84965.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657 2024-11-26T21:49:52,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741852_1028 (size=45) 2024-11-26T21:49:52,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741852_1028 (size=45) 2024-11-26T21:49:52,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741852_1028 (size=45) 2024-11-26T21:49:52,247 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ooo,1732657792023.8a6bb8489f79432157c419605ac84965.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:52,247 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 8a6bb8489f79432157c419605ac84965, disabling compactions & flushes 2024-11-26T21:49:52,247 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ooo,1732657792023.8a6bb8489f79432157c419605ac84965. 2024-11-26T21:49:52,247 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1732657792023.8a6bb8489f79432157c419605ac84965. 2024-11-26T21:49:52,247 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1732657792023.8a6bb8489f79432157c419605ac84965. after waiting 0 ms 2024-11-26T21:49:52,247 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1732657792023.8a6bb8489f79432157c419605ac84965. 2024-11-26T21:49:52,247 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ooo,1732657792023.8a6bb8489f79432157c419605ac84965. 2024-11-26T21:49:52,247 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8a6bb8489f79432157c419605ac84965: Waiting for close lock at 1732657792247Disabling compacts and flushes for region at 1732657792247Disabling writes for close at 1732657792247Writing region close event to WAL at 1732657792247Closed at 1732657792247 2024-11-26T21:49:52,248 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d1c5d2bc20a7b33de1c8b5a53f7fee57, NAME => 'test,ppp,1732657792023.d1c5d2bc20a7b33de1c8b5a53f7fee57.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657 2024-11-26T21:49:52,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741853_1029 (size=45) 2024-11-26T21:49:52,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741853_1029 (size=45) 2024-11-26T21:49:52,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741853_1029 (size=45) 2024-11-26T21:49:52,258 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ppp,1732657792023.d1c5d2bc20a7b33de1c8b5a53f7fee57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:52,258 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing d1c5d2bc20a7b33de1c8b5a53f7fee57, disabling compactions & flushes 2024-11-26T21:49:52,258 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ppp,1732657792023.d1c5d2bc20a7b33de1c8b5a53f7fee57. 2024-11-26T21:49:52,258 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732657792023.d1c5d2bc20a7b33de1c8b5a53f7fee57. 2024-11-26T21:49:52,258 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732657792023.d1c5d2bc20a7b33de1c8b5a53f7fee57. after waiting 0 ms 2024-11-26T21:49:52,258 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732657792023.d1c5d2bc20a7b33de1c8b5a53f7fee57. 2024-11-26T21:49:52,258 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ppp,1732657792023.d1c5d2bc20a7b33de1c8b5a53f7fee57. 2024-11-26T21:49:52,258 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for d1c5d2bc20a7b33de1c8b5a53f7fee57: Waiting for close lock at 1732657792258Disabling compacts and flushes for region at 1732657792258Disabling writes for close at 1732657792258Writing region close event to WAL at 1732657792258Closed at 1732657792258 2024-11-26T21:49:52,259 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 996589dbb2419ea602bc5e348cbd05df, NAME => 'test,qqq,1732657792023.996589dbb2419ea602bc5e348cbd05df.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657 2024-11-26T21:49:52,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741854_1030 (size=45) 2024-11-26T21:49:52,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741854_1030 (size=45) 2024-11-26T21:49:52,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741854_1030 (size=45) 2024-11-26T21:49:52,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T21:49:52,519 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,eee,1732657792023.651cf53d9624e6cc98e46d6578c71496.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:52,519 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 651cf53d9624e6cc98e46d6578c71496, disabling compactions & flushes 2024-11-26T21:49:52,519 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,eee,1732657792023.651cf53d9624e6cc98e46d6578c71496. 2024-11-26T21:49:52,519 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1732657792023.651cf53d9624e6cc98e46d6578c71496. 2024-11-26T21:49:52,519 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1732657792023.651cf53d9624e6cc98e46d6578c71496. after waiting 0 ms 2024-11-26T21:49:52,519 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1732657792023.651cf53d9624e6cc98e46d6578c71496. 2024-11-26T21:49:52,519 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,eee,1732657792023.651cf53d9624e6cc98e46d6578c71496. 2024-11-26T21:49:52,520 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 651cf53d9624e6cc98e46d6578c71496: Waiting for close lock at 1732657792519Disabling compacts and flushes for region at 1732657792519Disabling writes for close at 1732657792519Writing region close event to WAL at 1732657792519Closed at 1732657792519 2024-11-26T21:49:52,520 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 30cf2d9c6c6cf36bd61bb35f51df8f42, NAME => 'test,rrr,1732657792023.30cf2d9c6c6cf36bd61bb35f51df8f42.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657 2024-11-26T21:49:52,520 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ddd,1732657792023.7a7c374815f888db179beadaf2489ad7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:52,521 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 7a7c374815f888db179beadaf2489ad7, disabling compactions & flushes 2024-11-26T21:49:52,521 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ddd,1732657792023.7a7c374815f888db179beadaf2489ad7. 2024-11-26T21:49:52,521 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1732657792023.7a7c374815f888db179beadaf2489ad7. 2024-11-26T21:49:52,521 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1732657792023.7a7c374815f888db179beadaf2489ad7. after waiting 0 ms 2024-11-26T21:49:52,521 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1732657792023.7a7c374815f888db179beadaf2489ad7. 2024-11-26T21:49:52,521 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ddd,1732657792023.7a7c374815f888db179beadaf2489ad7. 2024-11-26T21:49:52,521 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 7a7c374815f888db179beadaf2489ad7: Waiting for close lock at 1732657792521Disabling compacts and flushes for region at 1732657792521Disabling writes for close at 1732657792521Writing region close event to WAL at 1732657792521Closed at 1732657792521 2024-11-26T21:49:52,522 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 2b8b8290cc9d27b8d226e5dc2b7d770d, NAME => 'test,sss,1732657792023.2b8b8290cc9d27b8d226e5dc2b7d770d.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657 2024-11-26T21:49:52,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741855_1031 (size=45) 2024-11-26T21:49:52,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741855_1031 (size=45) 2024-11-26T21:49:52,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741855_1031 (size=45) 2024-11-26T21:49:52,533 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,rrr,1732657792023.30cf2d9c6c6cf36bd61bb35f51df8f42.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:52,533 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 30cf2d9c6c6cf36bd61bb35f51df8f42, disabling compactions & flushes 2024-11-26T21:49:52,533 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,rrr,1732657792023.30cf2d9c6c6cf36bd61bb35f51df8f42. 2024-11-26T21:49:52,533 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732657792023.30cf2d9c6c6cf36bd61bb35f51df8f42. 2024-11-26T21:49:52,533 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732657792023.30cf2d9c6c6cf36bd61bb35f51df8f42. after waiting 0 ms 2024-11-26T21:49:52,533 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732657792023.30cf2d9c6c6cf36bd61bb35f51df8f42. 2024-11-26T21:49:52,533 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,rrr,1732657792023.30cf2d9c6c6cf36bd61bb35f51df8f42. 2024-11-26T21:49:52,533 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 30cf2d9c6c6cf36bd61bb35f51df8f42: Waiting for close lock at 1732657792533Disabling compacts and flushes for region at 1732657792533Disabling writes for close at 1732657792533Writing region close event to WAL at 1732657792533Closed at 1732657792533 2024-11-26T21:49:52,534 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 1ef98738dc2598149d89e210ccdb4e6f, NAME => 'test,ttt,1732657792023.1ef98738dc2598149d89e210ccdb4e6f.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657 2024-11-26T21:49:52,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741856_1032 (size=45) 2024-11-26T21:49:52,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741856_1032 (size=45) 2024-11-26T21:49:52,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741856_1032 (size=45) 2024-11-26T21:49:52,538 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,sss,1732657792023.2b8b8290cc9d27b8d226e5dc2b7d770d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:52,538 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 2b8b8290cc9d27b8d226e5dc2b7d770d, disabling compactions & flushes 2024-11-26T21:49:52,538 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,sss,1732657792023.2b8b8290cc9d27b8d226e5dc2b7d770d. 2024-11-26T21:49:52,538 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1732657792023.2b8b8290cc9d27b8d226e5dc2b7d770d. 2024-11-26T21:49:52,538 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1732657792023.2b8b8290cc9d27b8d226e5dc2b7d770d. after waiting 0 ms 2024-11-26T21:49:52,538 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1732657792023.2b8b8290cc9d27b8d226e5dc2b7d770d. 2024-11-26T21:49:52,538 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,sss,1732657792023.2b8b8290cc9d27b8d226e5dc2b7d770d. 2024-11-26T21:49:52,538 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 2b8b8290cc9d27b8d226e5dc2b7d770d: Waiting for close lock at 1732657792538Disabling compacts and flushes for region at 1732657792538Disabling writes for close at 1732657792538Writing region close event to WAL at 1732657792538Closed at 1732657792538 2024-11-26T21:49:52,539 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 317db83a078d27fdfa108a4454669013, NAME => 'test,uuu,1732657792023.317db83a078d27fdfa108a4454669013.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657 2024-11-26T21:49:52,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741857_1033 (size=45) 2024-11-26T21:49:52,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741857_1033 (size=45) 2024-11-26T21:49:52,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741857_1033 (size=45) 2024-11-26T21:49:52,548 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ttt,1732657792023.1ef98738dc2598149d89e210ccdb4e6f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:52,549 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 1ef98738dc2598149d89e210ccdb4e6f, disabling compactions & flushes 2024-11-26T21:49:52,549 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ttt,1732657792023.1ef98738dc2598149d89e210ccdb4e6f. 2024-11-26T21:49:52,549 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732657792023.1ef98738dc2598149d89e210ccdb4e6f. 2024-11-26T21:49:52,549 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732657792023.1ef98738dc2598149d89e210ccdb4e6f. after waiting 0 ms 2024-11-26T21:49:52,549 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732657792023.1ef98738dc2598149d89e210ccdb4e6f. 2024-11-26T21:49:52,549 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ttt,1732657792023.1ef98738dc2598149d89e210ccdb4e6f. 2024-11-26T21:49:52,549 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 1ef98738dc2598149d89e210ccdb4e6f: Waiting for close lock at 1732657792549Disabling compacts and flushes for region at 1732657792549Disabling writes for close at 1732657792549Writing region close event to WAL at 1732657792549Closed at 1732657792549 2024-11-26T21:49:52,550 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 21af909e3b8cf553daecee4e59c0898b, NAME => 'test,vvv,1732657792023.21af909e3b8cf553daecee4e59c0898b.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657 2024-11-26T21:49:52,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741858_1034 (size=45) 2024-11-26T21:49:52,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741858_1034 (size=45) 2024-11-26T21:49:52,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741858_1034 (size=45) 2024-11-26T21:49:52,553 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,uuu,1732657792023.317db83a078d27fdfa108a4454669013.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:52,554 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 317db83a078d27fdfa108a4454669013, disabling compactions & flushes 2024-11-26T21:49:52,554 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,uuu,1732657792023.317db83a078d27fdfa108a4454669013. 2024-11-26T21:49:52,554 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732657792023.317db83a078d27fdfa108a4454669013. 2024-11-26T21:49:52,554 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732657792023.317db83a078d27fdfa108a4454669013. after waiting 0 ms 2024-11-26T21:49:52,554 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732657792023.317db83a078d27fdfa108a4454669013. 2024-11-26T21:49:52,554 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,uuu,1732657792023.317db83a078d27fdfa108a4454669013. 2024-11-26T21:49:52,554 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 317db83a078d27fdfa108a4454669013: Waiting for close lock at 1732657792553Disabling compacts and flushes for region at 1732657792553Disabling writes for close at 1732657792554 (+1 ms)Writing region close event to WAL at 1732657792554Closed at 1732657792554 2024-11-26T21:49:52,554 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => f45cc001ffd649bcfbfe0f7af1630961, NAME => 'test,www,1732657792023.f45cc001ffd649bcfbfe0f7af1630961.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657 2024-11-26T21:49:52,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741859_1035 (size=45) 2024-11-26T21:49:52,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741859_1035 (size=45) 2024-11-26T21:49:52,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741859_1035 (size=45) 2024-11-26T21:49:52,563 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,vvv,1732657792023.21af909e3b8cf553daecee4e59c0898b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:52,563 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 21af909e3b8cf553daecee4e59c0898b, disabling compactions & flushes 2024-11-26T21:49:52,563 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,vvv,1732657792023.21af909e3b8cf553daecee4e59c0898b. 2024-11-26T21:49:52,563 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1732657792023.21af909e3b8cf553daecee4e59c0898b. 2024-11-26T21:49:52,563 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1732657792023.21af909e3b8cf553daecee4e59c0898b. after waiting 0 ms 2024-11-26T21:49:52,563 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1732657792023.21af909e3b8cf553daecee4e59c0898b. 2024-11-26T21:49:52,563 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,vvv,1732657792023.21af909e3b8cf553daecee4e59c0898b. 2024-11-26T21:49:52,563 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 21af909e3b8cf553daecee4e59c0898b: Waiting for close lock at 1732657792563Disabling compacts and flushes for region at 1732657792563Disabling writes for close at 1732657792563Writing region close event to WAL at 1732657792563Closed at 1732657792563 2024-11-26T21:49:52,564 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 0e9187292024abc7067d470edad3ae67, NAME => 'test,xxx,1732657792023.0e9187292024abc7067d470edad3ae67.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657 2024-11-26T21:49:52,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741860_1036 (size=45) 2024-11-26T21:49:52,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741860_1036 (size=45) 2024-11-26T21:49:52,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741860_1036 (size=45) 2024-11-26T21:49:52,572 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,www,1732657792023.f45cc001ffd649bcfbfe0f7af1630961.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:52,572 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing f45cc001ffd649bcfbfe0f7af1630961, disabling compactions & flushes 2024-11-26T21:49:52,573 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,www,1732657792023.f45cc001ffd649bcfbfe0f7af1630961. 2024-11-26T21:49:52,573 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1732657792023.f45cc001ffd649bcfbfe0f7af1630961. 2024-11-26T21:49:52,573 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1732657792023.f45cc001ffd649bcfbfe0f7af1630961. after waiting 0 ms 2024-11-26T21:49:52,573 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1732657792023.f45cc001ffd649bcfbfe0f7af1630961. 2024-11-26T21:49:52,573 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,www,1732657792023.f45cc001ffd649bcfbfe0f7af1630961. 2024-11-26T21:49:52,573 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for f45cc001ffd649bcfbfe0f7af1630961: Waiting for close lock at 1732657792572Disabling compacts and flushes for region at 1732657792572Disabling writes for close at 1732657792573 (+1 ms)Writing region close event to WAL at 1732657792573Closed at 1732657792573 2024-11-26T21:49:52,574 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1e7efa0e8c09fd1f0e1a765d7fc38181, NAME => 'test,yyy,1732657792023.1e7efa0e8c09fd1f0e1a765d7fc38181.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657 2024-11-26T21:49:52,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741861_1037 (size=45) 2024-11-26T21:49:52,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741861_1037 (size=45) 2024-11-26T21:49:52,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741861_1037 (size=45) 2024-11-26T21:49:52,581 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,xxx,1732657792023.0e9187292024abc7067d470edad3ae67.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:52,581 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 0e9187292024abc7067d470edad3ae67, disabling compactions & flushes 2024-11-26T21:49:52,581 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,xxx,1732657792023.0e9187292024abc7067d470edad3ae67. 2024-11-26T21:49:52,581 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1732657792023.0e9187292024abc7067d470edad3ae67. 2024-11-26T21:49:52,582 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1732657792023.0e9187292024abc7067d470edad3ae67. after waiting 0 ms 2024-11-26T21:49:52,582 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1732657792023.0e9187292024abc7067d470edad3ae67. 2024-11-26T21:49:52,582 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,xxx,1732657792023.0e9187292024abc7067d470edad3ae67. 2024-11-26T21:49:52,582 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 0e9187292024abc7067d470edad3ae67: Waiting for close lock at 1732657792581Disabling compacts and flushes for region at 1732657792581Disabling writes for close at 1732657792582 (+1 ms)Writing region close event to WAL at 1732657792582Closed at 1732657792582 2024-11-26T21:49:52,583 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 1062179709b27df36edcb7f6b054a4a6, NAME => 'test,zzz,1732657792023.1062179709b27df36edcb7f6b054a4a6.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657 2024-11-26T21:49:52,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741862_1038 (size=45) 2024-11-26T21:49:52,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741862_1038 (size=45) 2024-11-26T21:49:52,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741862_1038 (size=45) 2024-11-26T21:49:52,587 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,yyy,1732657792023.1e7efa0e8c09fd1f0e1a765d7fc38181.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:52,587 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 1e7efa0e8c09fd1f0e1a765d7fc38181, disabling compactions & flushes 2024-11-26T21:49:52,587 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,yyy,1732657792023.1e7efa0e8c09fd1f0e1a765d7fc38181. 2024-11-26T21:49:52,587 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732657792023.1e7efa0e8c09fd1f0e1a765d7fc38181. 2024-11-26T21:49:52,587 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732657792023.1e7efa0e8c09fd1f0e1a765d7fc38181. after waiting 0 ms 2024-11-26T21:49:52,587 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732657792023.1e7efa0e8c09fd1f0e1a765d7fc38181. 2024-11-26T21:49:52,587 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,yyy,1732657792023.1e7efa0e8c09fd1f0e1a765d7fc38181. 2024-11-26T21:49:52,587 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 1e7efa0e8c09fd1f0e1a765d7fc38181: Waiting for close lock at 1732657792587Disabling compacts and flushes for region at 1732657792587Disabling writes for close at 1732657792587Writing region close event to WAL at 1732657792587Closed at 1732657792587 2024-11-26T21:49:52,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741863_1039 (size=42) 2024-11-26T21:49:52,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741863_1039 (size=42) 2024-11-26T21:49:52,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741863_1039 (size=42) 2024-11-26T21:49:52,594 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,zzz,1732657792023.1062179709b27df36edcb7f6b054a4a6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:52,594 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 1062179709b27df36edcb7f6b054a4a6, disabling compactions & flushes 2024-11-26T21:49:52,594 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,zzz,1732657792023.1062179709b27df36edcb7f6b054a4a6. 2024-11-26T21:49:52,594 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1732657792023.1062179709b27df36edcb7f6b054a4a6. 2024-11-26T21:49:52,594 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1732657792023.1062179709b27df36edcb7f6b054a4a6. after waiting 0 ms 2024-11-26T21:49:52,594 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1732657792023.1062179709b27df36edcb7f6b054a4a6. 2024-11-26T21:49:52,594 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,zzz,1732657792023.1062179709b27df36edcb7f6b054a4a6. 2024-11-26T21:49:52,594 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 1062179709b27df36edcb7f6b054a4a6: Waiting for close lock at 1732657792594Disabling compacts and flushes for region at 1732657792594Disabling writes for close at 1732657792594Writing region close event to WAL at 1732657792594Closed at 1732657792594 2024-11-26T21:49:52,673 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,qqq,1732657792023.996589dbb2419ea602bc5e348cbd05df.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:52,673 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 996589dbb2419ea602bc5e348cbd05df, disabling compactions & flushes 2024-11-26T21:49:52,673 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,qqq,1732657792023.996589dbb2419ea602bc5e348cbd05df. 2024-11-26T21:49:52,673 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732657792023.996589dbb2419ea602bc5e348cbd05df. 2024-11-26T21:49:52,673 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732657792023.996589dbb2419ea602bc5e348cbd05df. after waiting 0 ms 2024-11-26T21:49:52,673 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732657792023.996589dbb2419ea602bc5e348cbd05df. 2024-11-26T21:49:52,673 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,qqq,1732657792023.996589dbb2419ea602bc5e348cbd05df. 2024-11-26T21:49:52,674 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 996589dbb2419ea602bc5e348cbd05df: Waiting for close lock at 1732657792673Disabling compacts and flushes for region at 1732657792673Disabling writes for close at 1732657792673Writing region close event to WAL at 1732657792673Closed at 1732657792673 2024-11-26T21:49:52,678 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T21:49:52,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T21:49:52,685 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1732657792023.36699c847bc0e6c6b06fd8533d3c9edb.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732657792679"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732657792679"}]},"ts":"1732657792679"} 2024-11-26T21:49:52,685 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1732657792023.34e003c01eb3f0a2e10e33152402164c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732657792679"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732657792679"}]},"ts":"1732657792679"} 2024-11-26T21:49:52,685 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1732657792023.bf200defbfcf83538e813f9fa91d9c35.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732657792679"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732657792679"}]},"ts":"1732657792679"} 2024-11-26T21:49:52,686 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1732657792023.f0a1ab633a90e6b5f5e9ba8ce6f66010.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732657792679"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732657792679"}]},"ts":"1732657792679"} 2024-11-26T21:49:52,686 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1732657792023.8be22ac0f4cc7a17874b9d10ba204518.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732657792679"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732657792679"}]},"ts":"1732657792679"} 2024-11-26T21:49:52,686 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1732657792023.a2d6b8d1f56582e3f52e3cfe7a116c1d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732657792679"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732657792679"}]},"ts":"1732657792679"} 2024-11-26T21:49:52,686 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1732657792023.d22f8ae31d7e7a4d0c284e3b9676a393.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732657792679"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732657792679"}]},"ts":"1732657792679"} 2024-11-26T21:49:52,686 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1732657792023.cdd2d12bbd55678c7f638b4ef0fde3da.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732657792679"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732657792679"}]},"ts":"1732657792679"} 2024-11-26T21:49:52,686 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1732657792023.334949367d92cc18a393737b69358f37.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732657792679"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732657792679"}]},"ts":"1732657792679"} 2024-11-26T21:49:52,687 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1732657792023.ac402503c294574c0e555c45a12f0a4e.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732657792679"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732657792679"}]},"ts":"1732657792679"} 2024-11-26T21:49:52,687 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1732657792023.06a1142ff2ac98bd39e6f73547d6aa98.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732657792679"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732657792679"}]},"ts":"1732657792679"} 2024-11-26T21:49:52,687 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1732657792023.c4145a9c4106377623484a3eb786fe8a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732657792679"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732657792679"}]},"ts":"1732657792679"} 2024-11-26T21:49:52,687 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1732657792023.8a6bb8489f79432157c419605ac84965.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732657792679"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732657792679"}]},"ts":"1732657792679"} 2024-11-26T21:49:52,687 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1732657792023.d1c5d2bc20a7b33de1c8b5a53f7fee57.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732657792679"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732657792679"}]},"ts":"1732657792679"} 2024-11-26T21:49:52,687 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1732657792023.651cf53d9624e6cc98e46d6578c71496.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732657792679"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732657792679"}]},"ts":"1732657792679"} 2024-11-26T21:49:52,688 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1732657792023.7a7c374815f888db179beadaf2489ad7.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732657792679"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732657792679"}]},"ts":"1732657792679"} 2024-11-26T21:49:52,688 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1732657792023.30cf2d9c6c6cf36bd61bb35f51df8f42.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732657792679"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732657792679"}]},"ts":"1732657792679"} 2024-11-26T21:49:52,688 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1732657792023.2b8b8290cc9d27b8d226e5dc2b7d770d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732657792679"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732657792679"}]},"ts":"1732657792679"} 2024-11-26T21:49:52,688 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1732657792023.1ef98738dc2598149d89e210ccdb4e6f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732657792679"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732657792679"}]},"ts":"1732657792679"} 2024-11-26T21:49:52,688 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1732657792023.317db83a078d27fdfa108a4454669013.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732657792679"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732657792679"}]},"ts":"1732657792679"} 2024-11-26T21:49:52,688 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1732657792023.21af909e3b8cf553daecee4e59c0898b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732657792679"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732657792679"}]},"ts":"1732657792679"} 2024-11-26T21:49:52,688 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1732657792023.f45cc001ffd649bcfbfe0f7af1630961.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732657792679"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732657792679"}]},"ts":"1732657792679"} 2024-11-26T21:49:52,688 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1732657792023.0e9187292024abc7067d470edad3ae67.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732657792679"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732657792679"}]},"ts":"1732657792679"} 2024-11-26T21:49:52,689 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1732657792023.1e7efa0e8c09fd1f0e1a765d7fc38181.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732657792679"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732657792679"}]},"ts":"1732657792679"} 2024-11-26T21:49:52,689 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1732657792023.1062179709b27df36edcb7f6b054a4a6.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732657792679"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732657792679"}]},"ts":"1732657792679"} 2024-11-26T21:49:52,689 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1732657792023.996589dbb2419ea602bc5e348cbd05df.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732657792679"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732657792679"}]},"ts":"1732657792679"} 2024-11-26T21:49:52,732 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-26T21:49:52,735 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T21:49:52,738 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732657792735"}]},"ts":"1732657792735"} 2024-11-26T21:49:52,743 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-11-26T21:49:52,744 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {828b3f090c91=0} racks are {/default-rack=0} 2024-11-26T21:49:52,760 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T21:49:52,760 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T21:49:52,760 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-26T21:49:52,760 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T21:49:52,760 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T21:49:52,760 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-26T21:49:52,760 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T21:49:52,760 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T21:49:52,760 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-26T21:49:52,760 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T21:49:52,763 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=36699c847bc0e6c6b06fd8533d3c9edb, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=34e003c01eb3f0a2e10e33152402164c, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7a7c374815f888db179beadaf2489ad7, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f0a1ab633a90e6b5f5e9ba8ce6f66010, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8be22ac0f4cc7a17874b9d10ba204518, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d22f8ae31d7e7a4d0c284e3b9676a393, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cdd2d12bbd55678c7f638b4ef0fde3da, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ac402503c294574c0e555c45a12f0a4e, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=06a1142ff2ac98bd39e6f73547d6aa98, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8a6bb8489f79432157c419605ac84965, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d1c5d2bc20a7b33de1c8b5a53f7fee57, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=996589dbb2419ea602bc5e348cbd05df, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=30cf2d9c6c6cf36bd61bb35f51df8f42, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1ef98738dc2598149d89e210ccdb4e6f, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=317db83a078d27fdfa108a4454669013, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f45cc001ffd649bcfbfe0f7af1630961, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0e9187292024abc7067d470edad3ae67, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1062179709b27df36edcb7f6b054a4a6, ASSIGN}] 2024-11-26T21:49:52,773 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1062179709b27df36edcb7f6b054a4a6, ASSIGN 2024-11-26T21:49:52,774 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1062179709b27df36edcb7f6b054a4a6, ASSIGN; state=OFFLINE, location=828b3f090c91,40359,1732657789839; forceNewPlan=false, retain=false 2024-11-26T21:49:52,776 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN 2024-11-26T21:49:52,777 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN; state=OFFLINE, location=828b3f090c91,46545,1732657789755; forceNewPlan=false, retain=false 2024-11-26T21:49:52,778 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0e9187292024abc7067d470edad3ae67, ASSIGN 2024-11-26T21:49:52,780 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0e9187292024abc7067d470edad3ae67, ASSIGN; state=OFFLINE, location=828b3f090c91,42273,1732657789877; forceNewPlan=false, retain=false 2024-11-26T21:49:52,781 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f45cc001ffd649bcfbfe0f7af1630961, ASSIGN 2024-11-26T21:49:52,782 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f45cc001ffd649bcfbfe0f7af1630961, ASSIGN; state=OFFLINE, location=828b3f090c91,40359,1732657789839; forceNewPlan=false, retain=false 2024-11-26T21:49:52,784 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN 2024-11-26T21:49:52,786 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN; state=OFFLINE, location=828b3f090c91,46545,1732657789755; forceNewPlan=false, retain=false 2024-11-26T21:49:52,787 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=317db83a078d27fdfa108a4454669013, ASSIGN 2024-11-26T21:49:52,788 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=317db83a078d27fdfa108a4454669013, ASSIGN; state=OFFLINE, location=828b3f090c91,42273,1732657789877; forceNewPlan=false, retain=false 2024-11-26T21:49:52,790 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1ef98738dc2598149d89e210ccdb4e6f, ASSIGN 2024-11-26T21:49:52,791 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1ef98738dc2598149d89e210ccdb4e6f, ASSIGN; state=OFFLINE, location=828b3f090c91,40359,1732657789839; forceNewPlan=false, retain=false 2024-11-26T21:49:52,792 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN 2024-11-26T21:49:52,793 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN; state=OFFLINE, location=828b3f090c91,46545,1732657789755; forceNewPlan=false, retain=false 2024-11-26T21:49:52,794 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=30cf2d9c6c6cf36bd61bb35f51df8f42, ASSIGN 2024-11-26T21:49:52,795 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=30cf2d9c6c6cf36bd61bb35f51df8f42, ASSIGN; state=OFFLINE, location=828b3f090c91,42273,1732657789877; forceNewPlan=false, retain=false 2024-11-26T21:49:52,797 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=996589dbb2419ea602bc5e348cbd05df, ASSIGN 2024-11-26T21:49:52,798 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=996589dbb2419ea602bc5e348cbd05df, ASSIGN; state=OFFLINE, location=828b3f090c91,42273,1732657789877; forceNewPlan=false, retain=false 2024-11-26T21:49:52,799 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d1c5d2bc20a7b33de1c8b5a53f7fee57, ASSIGN 2024-11-26T21:49:52,801 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d1c5d2bc20a7b33de1c8b5a53f7fee57, ASSIGN; state=OFFLINE, location=828b3f090c91,42273,1732657789877; forceNewPlan=false, retain=false 2024-11-26T21:49:52,802 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8a6bb8489f79432157c419605ac84965, ASSIGN 2024-11-26T21:49:52,803 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8a6bb8489f79432157c419605ac84965, ASSIGN; state=OFFLINE, location=828b3f090c91,40359,1732657789839; forceNewPlan=false, retain=false 2024-11-26T21:49:52,804 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN 2024-11-26T21:49:52,805 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN; state=OFFLINE, location=828b3f090c91,46545,1732657789755; forceNewPlan=false, retain=false 2024-11-26T21:49:52,807 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=06a1142ff2ac98bd39e6f73547d6aa98, ASSIGN 2024-11-26T21:49:52,808 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=06a1142ff2ac98bd39e6f73547d6aa98, ASSIGN; state=OFFLINE, location=828b3f090c91,42273,1732657789877; forceNewPlan=false, retain=false 2024-11-26T21:49:52,809 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ac402503c294574c0e555c45a12f0a4e, ASSIGN 2024-11-26T21:49:52,810 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ac402503c294574c0e555c45a12f0a4e, ASSIGN; state=OFFLINE, location=828b3f090c91,40359,1732657789839; forceNewPlan=false, retain=false 2024-11-26T21:49:52,811 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN 2024-11-26T21:49:52,812 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN; state=OFFLINE, location=828b3f090c91,46545,1732657789755; forceNewPlan=false, retain=false 2024-11-26T21:49:52,814 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cdd2d12bbd55678c7f638b4ef0fde3da, ASSIGN 2024-11-26T21:49:52,815 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=cdd2d12bbd55678c7f638b4ef0fde3da, ASSIGN; state=OFFLINE, location=828b3f090c91,42273,1732657789877; forceNewPlan=false, retain=false 2024-11-26T21:49:52,816 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d22f8ae31d7e7a4d0c284e3b9676a393, ASSIGN 2024-11-26T21:49:52,817 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d22f8ae31d7e7a4d0c284e3b9676a393, ASSIGN; state=OFFLINE, location=828b3f090c91,40359,1732657789839; forceNewPlan=false, retain=false 2024-11-26T21:49:52,819 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN 2024-11-26T21:49:52,820 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN; state=OFFLINE, location=828b3f090c91,46545,1732657789755; forceNewPlan=false, retain=false 2024-11-26T21:49:52,821 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8be22ac0f4cc7a17874b9d10ba204518, ASSIGN 2024-11-26T21:49:52,822 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8be22ac0f4cc7a17874b9d10ba204518, ASSIGN; state=OFFLINE, location=828b3f090c91,42273,1732657789877; forceNewPlan=false, retain=false 2024-11-26T21:49:52,824 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f0a1ab633a90e6b5f5e9ba8ce6f66010, ASSIGN 2024-11-26T21:49:52,825 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f0a1ab633a90e6b5f5e9ba8ce6f66010, ASSIGN; state=OFFLINE, location=828b3f090c91,40359,1732657789839; forceNewPlan=false, retain=false 2024-11-26T21:49:52,827 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN 2024-11-26T21:49:52,828 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN; state=OFFLINE, location=828b3f090c91,46545,1732657789755; forceNewPlan=false, retain=false 2024-11-26T21:49:52,830 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7a7c374815f888db179beadaf2489ad7, ASSIGN 2024-11-26T21:49:52,831 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7a7c374815f888db179beadaf2489ad7, ASSIGN; state=OFFLINE, location=828b3f090c91,40359,1732657789839; forceNewPlan=false, retain=false 2024-11-26T21:49:52,832 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=34e003c01eb3f0a2e10e33152402164c, ASSIGN 2024-11-26T21:49:52,834 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=34e003c01eb3f0a2e10e33152402164c, ASSIGN; state=OFFLINE, location=828b3f090c91,42273,1732657789877; forceNewPlan=false, retain=false 2024-11-26T21:49:52,835 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=36699c847bc0e6c6b06fd8533d3c9edb, ASSIGN 2024-11-26T21:49:52,837 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=36699c847bc0e6c6b06fd8533d3c9edb, ASSIGN; state=OFFLINE, location=828b3f090c91,40359,1732657789839; forceNewPlan=false, retain=false 2024-11-26T21:49:52,838 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN 2024-11-26T21:49:52,840 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN; state=OFFLINE, location=828b3f090c91,46545,1732657789755; forceNewPlan=false, retain=false 2024-11-26T21:49:52,929 INFO [828b3f090c91:44203 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-26T21:49:52,930 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=1e7efa0e8c09fd1f0e1a765d7fc38181, regionState=OPENING, regionLocation=828b3f090c91,46545,1732657789755 2024-11-26T21:49:52,932 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=21af909e3b8cf553daecee4e59c0898b, regionState=OPENING, regionLocation=828b3f090c91,46545,1732657789755 2024-11-26T21:49:52,933 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=2b8b8290cc9d27b8d226e5dc2b7d770d, regionState=OPENING, regionLocation=828b3f090c91,46545,1732657789755 2024-11-26T21:49:52,933 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=c4145a9c4106377623484a3eb786fe8a, regionState=OPENING, regionLocation=828b3f090c91,46545,1732657789755 2024-11-26T21:49:52,934 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=334949367d92cc18a393737b69358f37, regionState=OPENING, regionLocation=828b3f090c91,46545,1732657789755 2024-11-26T21:49:52,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN because future has completed 2024-11-26T21:49:52,935 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=a2d6b8d1f56582e3f52e3cfe7a116c1d, regionState=OPENING, regionLocation=828b3f090c91,46545,1732657789755 2024-11-26T21:49:52,936 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=651cf53d9624e6cc98e46d6578c71496, regionState=OPENING, regionLocation=828b3f090c91,46545,1732657789755 2024-11-26T21:49:52,937 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e7efa0e8c09fd1f0e1a765d7fc38181, server=828b3f090c91,46545,1732657789755}] 2024-11-26T21:49:52,939 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN because future has completed 2024-11-26T21:49:52,941 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN because future has completed 2024-11-26T21:49:52,945 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN because future has completed 2024-11-26T21:49:52,947 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN because future has completed 2024-11-26T21:49:52,949 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 334949367d92cc18a393737b69358f37, server=828b3f090c91,46545,1732657789755}] 2024-11-26T21:49:52,950 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN because future has completed 2024-11-26T21:49:52,956 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN because future has completed 2024-11-26T21:49:52,960 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,46545,1732657789755}] 2024-11-26T21:49:52,964 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure a2d6b8d1f56582e3f52e3cfe7a116c1d, server=828b3f090c91,46545,1732657789755}] 2024-11-26T21:49:52,968 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure c4145a9c4106377623484a3eb786fe8a, server=828b3f090c91,46545,1732657789755}] 2024-11-26T21:49:52,973 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b8b8290cc9d27b8d226e5dc2b7d770d, server=828b3f090c91,46545,1732657789755}] 2024-11-26T21:49:52,977 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21af909e3b8cf553daecee4e59c0898b, server=828b3f090c91,46545,1732657789755}] 2024-11-26T21:49:52,982 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=bf200defbfcf83538e813f9fa91d9c35, regionState=OPENING, regionLocation=828b3f090c91,46545,1732657789755 2024-11-26T21:49:52,983 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=0e9187292024abc7067d470edad3ae67, regionState=OPENING, regionLocation=828b3f090c91,42273,1732657789877 2024-11-26T21:49:52,983 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=317db83a078d27fdfa108a4454669013, regionState=OPENING, regionLocation=828b3f090c91,42273,1732657789877 2024-11-26T21:49:52,984 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=30cf2d9c6c6cf36bd61bb35f51df8f42, regionState=OPENING, regionLocation=828b3f090c91,42273,1732657789877 2024-11-26T21:49:52,985 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=996589dbb2419ea602bc5e348cbd05df, regionState=OPENING, regionLocation=828b3f090c91,42273,1732657789877 2024-11-26T21:49:52,985 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN because future has completed 2024-11-26T21:49:52,985 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=d1c5d2bc20a7b33de1c8b5a53f7fee57, regionState=OPENING, regionLocation=828b3f090c91,42273,1732657789877 2024-11-26T21:49:52,986 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure bf200defbfcf83538e813f9fa91d9c35, server=828b3f090c91,46545,1732657789755}] 2024-11-26T21:49:52,988 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0e9187292024abc7067d470edad3ae67, ASSIGN because future has completed 2024-11-26T21:49:52,990 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=317db83a078d27fdfa108a4454669013, ASSIGN because future has completed 2024-11-26T21:49:52,991 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=30cf2d9c6c6cf36bd61bb35f51df8f42, ASSIGN because future has completed 2024-11-26T21:49:52,993 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 30cf2d9c6c6cf36bd61bb35f51df8f42, server=828b3f090c91,42273,1732657789877}] 2024-11-26T21:49:52,993 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=996589dbb2419ea602bc5e348cbd05df, ASSIGN because future has completed 2024-11-26T21:49:52,995 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d1c5d2bc20a7b33de1c8b5a53f7fee57, ASSIGN because future has completed 2024-11-26T21:49:52,998 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1c5d2bc20a7b33de1c8b5a53f7fee57, server=828b3f090c91,42273,1732657789877}] 2024-11-26T21:49:53,003 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 996589dbb2419ea602bc5e348cbd05df, server=828b3f090c91,42273,1732657789877}] 2024-11-26T21:49:53,008 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 317db83a078d27fdfa108a4454669013, server=828b3f090c91,42273,1732657789877}] 2024-11-26T21:49:53,014 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0e9187292024abc7067d470edad3ae67, server=828b3f090c91,42273,1732657789877}] 2024-11-26T21:49:53,018 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=06a1142ff2ac98bd39e6f73547d6aa98, regionState=OPENING, regionLocation=828b3f090c91,42273,1732657789877 2024-11-26T21:49:53,019 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=cdd2d12bbd55678c7f638b4ef0fde3da, regionState=OPENING, regionLocation=828b3f090c91,42273,1732657789877 2024-11-26T21:49:53,020 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=8be22ac0f4cc7a17874b9d10ba204518, regionState=OPENING, regionLocation=828b3f090c91,42273,1732657789877 2024-11-26T21:49:53,021 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=34e003c01eb3f0a2e10e33152402164c, regionState=OPENING, regionLocation=828b3f090c91,42273,1732657789877 2024-11-26T21:49:53,021 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=1062179709b27df36edcb7f6b054a4a6, regionState=OPENING, regionLocation=828b3f090c91,40359,1732657789839 2024-11-26T21:49:53,022 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=f45cc001ffd649bcfbfe0f7af1630961, regionState=OPENING, regionLocation=828b3f090c91,40359,1732657789839 2024-11-26T21:49:53,022 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=06a1142ff2ac98bd39e6f73547d6aa98, ASSIGN because future has completed 2024-11-26T21:49:53,023 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06a1142ff2ac98bd39e6f73547d6aa98, server=828b3f090c91,42273,1732657789877}] 2024-11-26T21:49:53,025 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=cdd2d12bbd55678c7f638b4ef0fde3da, ASSIGN because future has completed 2024-11-26T21:49:53,026 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8be22ac0f4cc7a17874b9d10ba204518, ASSIGN because future has completed 2024-11-26T21:49:53,029 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=34e003c01eb3f0a2e10e33152402164c, ASSIGN because future has completed 2024-11-26T21:49:53,030 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 34e003c01eb3f0a2e10e33152402164c, server=828b3f090c91,42273,1732657789877}] 2024-11-26T21:49:53,031 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1062179709b27df36edcb7f6b054a4a6, ASSIGN because future has completed 2024-11-26T21:49:53,034 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f45cc001ffd649bcfbfe0f7af1630961, ASSIGN because future has completed 2024-11-26T21:49:53,037 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure f45cc001ffd649bcfbfe0f7af1630961, server=828b3f090c91,40359,1732657789839}] 2024-11-26T21:49:53,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1062179709b27df36edcb7f6b054a4a6, server=828b3f090c91,40359,1732657789839}] 2024-11-26T21:49:53,046 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8be22ac0f4cc7a17874b9d10ba204518, server=828b3f090c91,42273,1732657789877}] 2024-11-26T21:49:53,050 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure cdd2d12bbd55678c7f638b4ef0fde3da, server=828b3f090c91,42273,1732657789877}] 2024-11-26T21:49:53,054 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=1ef98738dc2598149d89e210ccdb4e6f, regionState=OPENING, regionLocation=828b3f090c91,40359,1732657789839 2024-11-26T21:49:53,055 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=8a6bb8489f79432157c419605ac84965, regionState=OPENING, regionLocation=828b3f090c91,40359,1732657789839 2024-11-26T21:49:53,055 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=ac402503c294574c0e555c45a12f0a4e, regionState=OPENING, regionLocation=828b3f090c91,40359,1732657789839 2024-11-26T21:49:53,056 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=d22f8ae31d7e7a4d0c284e3b9676a393, regionState=OPENING, regionLocation=828b3f090c91,40359,1732657789839 2024-11-26T21:49:53,057 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=f0a1ab633a90e6b5f5e9ba8ce6f66010, regionState=OPENING, regionLocation=828b3f090c91,40359,1732657789839 2024-11-26T21:49:53,057 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1ef98738dc2598149d89e210ccdb4e6f, ASSIGN because future has completed 2024-11-26T21:49:53,057 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=7a7c374815f888db179beadaf2489ad7, regionState=OPENING, regionLocation=828b3f090c91,40359,1732657789839 2024-11-26T21:49:53,058 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ef98738dc2598149d89e210ccdb4e6f, server=828b3f090c91,40359,1732657789839}] 2024-11-26T21:49:53,059 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8a6bb8489f79432157c419605ac84965, ASSIGN because future has completed 2024-11-26T21:49:53,060 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ac402503c294574c0e555c45a12f0a4e, ASSIGN because future has completed 2024-11-26T21:49:53,062 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d22f8ae31d7e7a4d0c284e3b9676a393, ASSIGN because future has completed 2024-11-26T21:49:53,063 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f0a1ab633a90e6b5f5e9ba8ce6f66010, ASSIGN because future has completed 2024-11-26T21:49:53,064 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure f0a1ab633a90e6b5f5e9ba8ce6f66010, server=828b3f090c91,40359,1732657789839}] 2024-11-26T21:49:53,064 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7a7c374815f888db179beadaf2489ad7, ASSIGN because future has completed 2024-11-26T21:49:53,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7a7c374815f888db179beadaf2489ad7, server=828b3f090c91,40359,1732657789839}] 2024-11-26T21:49:53,071 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure d22f8ae31d7e7a4d0c284e3b9676a393, server=828b3f090c91,40359,1732657789839}] 2024-11-26T21:49:53,075 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure ac402503c294574c0e555c45a12f0a4e, server=828b3f090c91,40359,1732657789839}] 2024-11-26T21:49:53,079 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a6bb8489f79432157c419605ac84965, server=828b3f090c91,40359,1732657789839}] 2024-11-26T21:49:53,084 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=36699c847bc0e6c6b06fd8533d3c9edb, regionState=OPENING, regionLocation=828b3f090c91,40359,1732657789839 2024-11-26T21:49:53,087 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=36699c847bc0e6c6b06fd8533d3c9edb, ASSIGN because future has completed 2024-11-26T21:49:53,087 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 36699c847bc0e6c6b06fd8533d3c9edb, server=828b3f090c91,40359,1732657789839}] 2024-11-26T21:49:53,102 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,hhh,1732657792023.a2d6b8d1f56582e3f52e3cfe7a116c1d. 2024-11-26T21:49:53,102 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => a2d6b8d1f56582e3f52e3cfe7a116c1d, NAME => 'test,hhh,1732657792023.a2d6b8d1f56582e3f52e3cfe7a116c1d.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-26T21:49:53,103 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a2d6b8d1f56582e3f52e3cfe7a116c1d 2024-11-26T21:49:53,103 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,hhh,1732657792023.a2d6b8d1f56582e3f52e3cfe7a116c1d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:53,103 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for a2d6b8d1f56582e3f52e3cfe7a116c1d 2024-11-26T21:49:53,103 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for a2d6b8d1f56582e3f52e3cfe7a116c1d 2024-11-26T21:49:53,106 INFO [StoreOpener-a2d6b8d1f56582e3f52e3cfe7a116c1d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a2d6b8d1f56582e3f52e3cfe7a116c1d 2024-11-26T21:49:53,108 INFO [StoreOpener-a2d6b8d1f56582e3f52e3cfe7a116c1d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2d6b8d1f56582e3f52e3cfe7a116c1d columnFamilyName family 2024-11-26T21:49:53,108 DEBUG [StoreOpener-a2d6b8d1f56582e3f52e3cfe7a116c1d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:53,109 INFO [StoreOpener-a2d6b8d1f56582e3f52e3cfe7a116c1d-1 {}] regionserver.HStore(327): Store=a2d6b8d1f56582e3f52e3cfe7a116c1d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:49:53,109 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for a2d6b8d1f56582e3f52e3cfe7a116c1d 2024-11-26T21:49:53,111 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/a2d6b8d1f56582e3f52e3cfe7a116c1d 2024-11-26T21:49:53,111 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/a2d6b8d1f56582e3f52e3cfe7a116c1d 2024-11-26T21:49:53,112 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for a2d6b8d1f56582e3f52e3cfe7a116c1d 2024-11-26T21:49:53,112 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for a2d6b8d1f56582e3f52e3cfe7a116c1d 2024-11-26T21:49:53,114 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for a2d6b8d1f56582e3f52e3cfe7a116c1d 2024-11-26T21:49:53,116 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/a2d6b8d1f56582e3f52e3cfe7a116c1d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T21:49:53,117 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened a2d6b8d1f56582e3f52e3cfe7a116c1d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59802925, jitterRate=-0.10886697471141815}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T21:49:53,117 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a2d6b8d1f56582e3f52e3cfe7a116c1d 2024-11-26T21:49:53,118 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for a2d6b8d1f56582e3f52e3cfe7a116c1d: Running coprocessor pre-open hook at 1732657793103Writing region info on filesystem at 1732657793103Initializing all the Stores at 1732657793105 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657793105Cleaning up temporary data from old regions at 1732657793112 (+7 ms)Running coprocessor post-open hooks at 1732657793117 (+5 ms)Region opened successfully at 1732657793118 (+1 ms) 2024-11-26T21:49:53,120 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1732657792023.a2d6b8d1f56582e3f52e3cfe7a116c1d., pid=34, masterSystemTime=1732657793095 2024-11-26T21:49:53,122 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1732657792023.a2d6b8d1f56582e3f52e3cfe7a116c1d. 2024-11-26T21:49:53,122 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,hhh,1732657792023.a2d6b8d1f56582e3f52e3cfe7a116c1d. 2024-11-26T21:49:53,123 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,,1732657792023.bf200defbfcf83538e813f9fa91d9c35. 2024-11-26T21:49:53,123 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => bf200defbfcf83538e813f9fa91d9c35, NAME => 'test,,1732657792023.bf200defbfcf83538e813f9fa91d9c35.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-26T21:49:53,123 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=a2d6b8d1f56582e3f52e3cfe7a116c1d, regionState=OPEN, openSeqNum=2, regionLocation=828b3f090c91,46545,1732657789755 2024-11-26T21:49:53,123 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test bf200defbfcf83538e813f9fa91d9c35 2024-11-26T21:49:53,123 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,,1732657792023.bf200defbfcf83538e813f9fa91d9c35.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:53,124 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for bf200defbfcf83538e813f9fa91d9c35 2024-11-26T21:49:53,124 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for bf200defbfcf83538e813f9fa91d9c35 2024-11-26T21:49:53,126 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure a2d6b8d1f56582e3f52e3cfe7a116c1d, server=828b3f090c91,46545,1732657789755 because future has completed 2024-11-26T21:49:53,126 INFO [StoreOpener-bf200defbfcf83538e813f9fa91d9c35-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region bf200defbfcf83538e813f9fa91d9c35 2024-11-26T21:49:53,129 INFO [StoreOpener-bf200defbfcf83538e813f9fa91d9c35-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf200defbfcf83538e813f9fa91d9c35 columnFamilyName family 2024-11-26T21:49:53,129 DEBUG [StoreOpener-bf200defbfcf83538e813f9fa91d9c35-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:53,130 INFO [StoreOpener-bf200defbfcf83538e813f9fa91d9c35-1 {}] regionserver.HStore(327): Store=bf200defbfcf83538e813f9fa91d9c35/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:49:53,130 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for bf200defbfcf83538e813f9fa91d9c35 2024-11-26T21:49:53,132 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=12 2024-11-26T21:49:53,132 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/bf200defbfcf83538e813f9fa91d9c35 2024-11-26T21:49:53,132 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure a2d6b8d1f56582e3f52e3cfe7a116c1d, server=828b3f090c91,46545,1732657789755 in 163 msec 2024-11-26T21:49:53,132 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/bf200defbfcf83538e813f9fa91d9c35 2024-11-26T21:49:53,133 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for bf200defbfcf83538e813f9fa91d9c35 2024-11-26T21:49:53,133 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for bf200defbfcf83538e813f9fa91d9c35 2024-11-26T21:49:53,135 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN in 369 msec 2024-11-26T21:49:53,136 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for bf200defbfcf83538e813f9fa91d9c35 2024-11-26T21:49:53,139 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/bf200defbfcf83538e813f9fa91d9c35/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T21:49:53,140 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened bf200defbfcf83538e813f9fa91d9c35; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66655417, jitterRate=-0.006756886839866638}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T21:49:53,141 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bf200defbfcf83538e813f9fa91d9c35 2024-11-26T21:49:53,141 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for bf200defbfcf83538e813f9fa91d9c35: Running coprocessor pre-open hook at 1732657793124Writing region info on filesystem at 1732657793124Initializing all the Stores at 1732657793126 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657793126Cleaning up temporary data from old regions at 1732657793133 (+7 ms)Running coprocessor post-open hooks at 1732657793141 (+8 ms)Region opened successfully at 1732657793141 2024-11-26T21:49:53,142 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1732657792023.bf200defbfcf83538e813f9fa91d9c35., pid=38, masterSystemTime=1732657793095 2024-11-26T21:49:53,145 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1732657792023.bf200defbfcf83538e813f9fa91d9c35. 2024-11-26T21:49:53,145 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,,1732657792023.bf200defbfcf83538e813f9fa91d9c35. 2024-11-26T21:49:53,145 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,nnn,1732657792023.c4145a9c4106377623484a3eb786fe8a. 2024-11-26T21:49:53,146 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => c4145a9c4106377623484a3eb786fe8a, NAME => 'test,nnn,1732657792023.c4145a9c4106377623484a3eb786fe8a.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-26T21:49:53,146 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=bf200defbfcf83538e813f9fa91d9c35, regionState=OPEN, openSeqNum=2, regionLocation=828b3f090c91,46545,1732657789755 2024-11-26T21:49:53,146 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c4145a9c4106377623484a3eb786fe8a 2024-11-26T21:49:53,146 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,nnn,1732657792023.c4145a9c4106377623484a3eb786fe8a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:53,147 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for c4145a9c4106377623484a3eb786fe8a 2024-11-26T21:49:53,147 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for c4145a9c4106377623484a3eb786fe8a 2024-11-26T21:49:53,147 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T21:49:53,149 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44695, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T21:49:53,149 INFO [StoreOpener-c4145a9c4106377623484a3eb786fe8a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c4145a9c4106377623484a3eb786fe8a 2024-11-26T21:49:53,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure bf200defbfcf83538e813f9fa91d9c35, server=828b3f090c91,46545,1732657789755 because future has completed 2024-11-26T21:49:53,152 INFO [StoreOpener-c4145a9c4106377623484a3eb786fe8a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c4145a9c4106377623484a3eb786fe8a columnFamilyName family 2024-11-26T21:49:53,152 DEBUG [StoreOpener-c4145a9c4106377623484a3eb786fe8a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:53,153 INFO [StoreOpener-c4145a9c4106377623484a3eb786fe8a-1 {}] regionserver.HStore(327): Store=c4145a9c4106377623484a3eb786fe8a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:49:53,156 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for c4145a9c4106377623484a3eb786fe8a 2024-11-26T21:49:53,158 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/c4145a9c4106377623484a3eb786fe8a 2024-11-26T21:49:53,159 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/c4145a9c4106377623484a3eb786fe8a 2024-11-26T21:49:53,159 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=5 2024-11-26T21:49:53,160 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure bf200defbfcf83538e813f9fa91d9c35, server=828b3f090c91,46545,1732657789755 in 169 msec 2024-11-26T21:49:53,160 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for c4145a9c4106377623484a3eb786fe8a 2024-11-26T21:49:53,160 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for c4145a9c4106377623484a3eb786fe8a 2024-11-26T21:49:53,162 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,mmm,1732657792023.06a1142ff2ac98bd39e6f73547d6aa98. 2024-11-26T21:49:53,162 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 06a1142ff2ac98bd39e6f73547d6aa98, NAME => 'test,mmm,1732657792023.06a1142ff2ac98bd39e6f73547d6aa98.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-26T21:49:53,163 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 06a1142ff2ac98bd39e6f73547d6aa98 2024-11-26T21:49:53,163 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,mmm,1732657792023.06a1142ff2ac98bd39e6f73547d6aa98.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:53,164 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 06a1142ff2ac98bd39e6f73547d6aa98 2024-11-26T21:49:53,164 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 06a1142ff2ac98bd39e6f73547d6aa98 2024-11-26T21:49:53,164 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for c4145a9c4106377623484a3eb786fe8a 2024-11-26T21:49:53,165 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN in 397 msec 2024-11-26T21:49:53,167 INFO [StoreOpener-06a1142ff2ac98bd39e6f73547d6aa98-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 06a1142ff2ac98bd39e6f73547d6aa98 2024-11-26T21:49:53,167 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/c4145a9c4106377623484a3eb786fe8a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T21:49:53,168 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened c4145a9c4106377623484a3eb786fe8a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69925468, jitterRate=0.041970670223236084}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T21:49:53,168 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c4145a9c4106377623484a3eb786fe8a 2024-11-26T21:49:53,169 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for c4145a9c4106377623484a3eb786fe8a: Running coprocessor pre-open hook at 1732657793147Writing region info on filesystem at 1732657793147Initializing all the Stores at 1732657793149 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657793149Cleaning up temporary data from old regions at 1732657793160 (+11 ms)Running coprocessor post-open hooks at 1732657793168 (+8 ms)Region opened successfully at 1732657793168 2024-11-26T21:49:53,170 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1732657792023.c4145a9c4106377623484a3eb786fe8a., pid=35, masterSystemTime=1732657793095 2024-11-26T21:49:53,170 INFO [StoreOpener-06a1142ff2ac98bd39e6f73547d6aa98-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06a1142ff2ac98bd39e6f73547d6aa98 columnFamilyName family 2024-11-26T21:49:53,170 DEBUG [StoreOpener-06a1142ff2ac98bd39e6f73547d6aa98-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:53,171 INFO [StoreOpener-06a1142ff2ac98bd39e6f73547d6aa98-1 {}] regionserver.HStore(327): Store=06a1142ff2ac98bd39e6f73547d6aa98/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:49:53,171 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 06a1142ff2ac98bd39e6f73547d6aa98 2024-11-26T21:49:53,172 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1732657792023.c4145a9c4106377623484a3eb786fe8a. 2024-11-26T21:49:53,173 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/06a1142ff2ac98bd39e6f73547d6aa98 2024-11-26T21:49:53,173 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,nnn,1732657792023.c4145a9c4106377623484a3eb786fe8a. 2024-11-26T21:49:53,173 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,eee,1732657792023.651cf53d9624e6cc98e46d6578c71496. 2024-11-26T21:49:53,173 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 651cf53d9624e6cc98e46d6578c71496, NAME => 'test,eee,1732657792023.651cf53d9624e6cc98e46d6578c71496.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-26T21:49:53,173 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/06a1142ff2ac98bd39e6f73547d6aa98 2024-11-26T21:49:53,173 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=c4145a9c4106377623484a3eb786fe8a, regionState=OPEN, openSeqNum=2, regionLocation=828b3f090c91,46545,1732657789755 2024-11-26T21:49:53,174 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 651cf53d9624e6cc98e46d6578c71496 2024-11-26T21:49:53,174 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,eee,1732657792023.651cf53d9624e6cc98e46d6578c71496.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:53,174 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 06a1142ff2ac98bd39e6f73547d6aa98 2024-11-26T21:49:53,174 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 651cf53d9624e6cc98e46d6578c71496 2024-11-26T21:49:53,174 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 06a1142ff2ac98bd39e6f73547d6aa98 2024-11-26T21:49:53,174 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 651cf53d9624e6cc98e46d6578c71496 2024-11-26T21:49:53,177 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure c4145a9c4106377623484a3eb786fe8a, server=828b3f090c91,46545,1732657789755 because future has completed 2024-11-26T21:49:53,177 INFO [StoreOpener-651cf53d9624e6cc98e46d6578c71496-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 651cf53d9624e6cc98e46d6578c71496 2024-11-26T21:49:53,177 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 06a1142ff2ac98bd39e6f73547d6aa98 2024-11-26T21:49:53,179 INFO [StoreOpener-651cf53d9624e6cc98e46d6578c71496-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 651cf53d9624e6cc98e46d6578c71496 columnFamilyName family 2024-11-26T21:49:53,179 DEBUG [StoreOpener-651cf53d9624e6cc98e46d6578c71496-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:53,180 INFO [StoreOpener-651cf53d9624e6cc98e46d6578c71496-1 {}] regionserver.HStore(327): Store=651cf53d9624e6cc98e46d6578c71496/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:49:53,181 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 651cf53d9624e6cc98e46d6578c71496 2024-11-26T21:49:53,181 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/06a1142ff2ac98bd39e6f73547d6aa98/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T21:49:53,181 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 06a1142ff2ac98bd39e6f73547d6aa98; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70826047, jitterRate=0.05539034307003021}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T21:49:53,181 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 06a1142ff2ac98bd39e6f73547d6aa98 2024-11-26T21:49:53,181 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 06a1142ff2ac98bd39e6f73547d6aa98: Running coprocessor pre-open hook at 1732657793164Writing region info on filesystem at 1732657793164Initializing all the Stores at 1732657793165 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657793165Cleaning up temporary data from old regions at 1732657793174 (+9 ms)Running coprocessor post-open hooks at 1732657793181 (+7 ms)Region opened successfully at 1732657793181 2024-11-26T21:49:53,182 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/651cf53d9624e6cc98e46d6578c71496 2024-11-26T21:49:53,182 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/651cf53d9624e6cc98e46d6578c71496 2024-11-26T21:49:53,183 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1732657792023.06a1142ff2ac98bd39e6f73547d6aa98., pid=44, masterSystemTime=1732657793146 2024-11-26T21:49:53,183 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 651cf53d9624e6cc98e46d6578c71496 2024-11-26T21:49:53,183 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 651cf53d9624e6cc98e46d6578c71496 2024-11-26T21:49:53,185 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 651cf53d9624e6cc98e46d6578c71496 2024-11-26T21:49:53,186 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1732657792023.06a1142ff2ac98bd39e6f73547d6aa98. 2024-11-26T21:49:53,186 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,mmm,1732657792023.06a1142ff2ac98bd39e6f73547d6aa98. 2024-11-26T21:49:53,186 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,ccc,1732657792023.34e003c01eb3f0a2e10e33152402164c. 2024-11-26T21:49:53,186 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => 34e003c01eb3f0a2e10e33152402164c, NAME => 'test,ccc,1732657792023.34e003c01eb3f0a2e10e33152402164c.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-26T21:49:53,187 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 34e003c01eb3f0a2e10e33152402164c 2024-11-26T21:49:53,187 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,ccc,1732657792023.34e003c01eb3f0a2e10e33152402164c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:53,187 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for 34e003c01eb3f0a2e10e33152402164c 2024-11-26T21:49:53,187 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for 34e003c01eb3f0a2e10e33152402164c 2024-11-26T21:49:53,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=18 2024-11-26T21:49:53,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure c4145a9c4106377623484a3eb786fe8a, server=828b3f090c91,46545,1732657789755 in 210 msec 2024-11-26T21:49:53,189 INFO [StoreOpener-34e003c01eb3f0a2e10e33152402164c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 34e003c01eb3f0a2e10e33152402164c 2024-11-26T21:49:53,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T21:49:53,190 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/651cf53d9624e6cc98e46d6578c71496/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T21:49:53,191 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T21:49:53,192 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN in 425 msec 2024-11-26T21:49:53,193 INFO [StoreOpener-34e003c01eb3f0a2e10e33152402164c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 34e003c01eb3f0a2e10e33152402164c columnFamilyName family 2024-11-26T21:49:53,193 DEBUG [StoreOpener-34e003c01eb3f0a2e10e33152402164c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:53,193 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 651cf53d9624e6cc98e46d6578c71496; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64198941, jitterRate=-0.04336123168468475}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T21:49:53,194 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 651cf53d9624e6cc98e46d6578c71496 2024-11-26T21:49:53,194 INFO [StoreOpener-34e003c01eb3f0a2e10e33152402164c-1 {}] regionserver.HStore(327): Store=34e003c01eb3f0a2e10e33152402164c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:49:53,194 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 651cf53d9624e6cc98e46d6578c71496: Running coprocessor pre-open hook at 1732657793174Writing region info on filesystem at 1732657793174Initializing all the Stores at 1732657793176 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657793176Cleaning up temporary data from old regions at 1732657793183 (+7 ms)Running coprocessor post-open hooks at 1732657793194 (+11 ms)Region opened successfully at 1732657793194 2024-11-26T21:49:53,194 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for 34e003c01eb3f0a2e10e33152402164c 2024-11-26T21:49:53,196 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=06a1142ff2ac98bd39e6f73547d6aa98, regionState=OPEN, openSeqNum=2, regionLocation=828b3f090c91,42273,1732657789877 2024-11-26T21:49:53,196 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/34e003c01eb3f0a2e10e33152402164c 2024-11-26T21:49:53,197 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/34e003c01eb3f0a2e10e33152402164c 2024-11-26T21:49:53,198 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1732657792023.651cf53d9624e6cc98e46d6578c71496., pid=33, masterSystemTime=1732657793095 2024-11-26T21:49:53,198 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for 34e003c01eb3f0a2e10e33152402164c 2024-11-26T21:49:53,198 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for 34e003c01eb3f0a2e10e33152402164c 2024-11-26T21:49:53,200 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06a1142ff2ac98bd39e6f73547d6aa98, server=828b3f090c91,42273,1732657789877 because future has completed 2024-11-26T21:49:53,201 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for 34e003c01eb3f0a2e10e33152402164c 2024-11-26T21:49:53,201 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1732657792023.651cf53d9624e6cc98e46d6578c71496. 2024-11-26T21:49:53,202 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,eee,1732657792023.651cf53d9624e6cc98e46d6578c71496. 2024-11-26T21:49:53,202 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,kkk,1732657792023.334949367d92cc18a393737b69358f37. 2024-11-26T21:49:53,202 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 334949367d92cc18a393737b69358f37, NAME => 'test,kkk,1732657792023.334949367d92cc18a393737b69358f37.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-26T21:49:53,203 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 334949367d92cc18a393737b69358f37 2024-11-26T21:49:53,203 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,kkk,1732657792023.334949367d92cc18a393737b69358f37.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:53,203 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 334949367d92cc18a393737b69358f37 2024-11-26T21:49:53,203 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 334949367d92cc18a393737b69358f37 2024-11-26T21:49:53,206 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57761, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T21:49:53,206 INFO [StoreOpener-334949367d92cc18a393737b69358f37-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 334949367d92cc18a393737b69358f37 2024-11-26T21:49:53,206 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/34e003c01eb3f0a2e10e33152402164c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T21:49:53,207 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=17 2024-11-26T21:49:53,207 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 06a1142ff2ac98bd39e6f73547d6aa98, server=828b3f090c91,42273,1732657789877 in 179 msec 2024-11-26T21:49:53,207 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened 34e003c01eb3f0a2e10e33152402164c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67691497, jitterRate=0.008681908249855042}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T21:49:53,207 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 34e003c01eb3f0a2e10e33152402164c 2024-11-26T21:49:53,208 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for 34e003c01eb3f0a2e10e33152402164c: Running coprocessor pre-open hook at 1732657793187Writing region info on filesystem at 1732657793187Initializing all the Stores at 1732657793188 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657793189 (+1 ms)Cleaning up temporary data from old regions at 1732657793198 (+9 ms)Running coprocessor post-open hooks at 1732657793207 (+9 ms)Region opened successfully at 1732657793207 2024-11-26T21:49:53,209 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1732657792023.34e003c01eb3f0a2e10e33152402164c., pid=45, masterSystemTime=1732657793146 2024-11-26T21:49:53,209 INFO [StoreOpener-334949367d92cc18a393737b69358f37-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 334949367d92cc18a393737b69358f37 columnFamilyName family 2024-11-26T21:49:53,209 DEBUG [StoreOpener-334949367d92cc18a393737b69358f37-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:53,211 INFO [StoreOpener-334949367d92cc18a393737b69358f37-1 {}] regionserver.HStore(327): Store=334949367d92cc18a393737b69358f37/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:49:53,211 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=06a1142ff2ac98bd39e6f73547d6aa98, ASSIGN in 444 msec 2024-11-26T21:49:53,211 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 334949367d92cc18a393737b69358f37 2024-11-26T21:49:53,212 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1732657792023.34e003c01eb3f0a2e10e33152402164c. 2024-11-26T21:49:53,212 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,ccc,1732657792023.34e003c01eb3f0a2e10e33152402164c. 2024-11-26T21:49:53,212 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,ggg,1732657792023.8be22ac0f4cc7a17874b9d10ba204518. 2024-11-26T21:49:53,212 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => 8be22ac0f4cc7a17874b9d10ba204518, NAME => 'test,ggg,1732657792023.8be22ac0f4cc7a17874b9d10ba204518.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-26T21:49:53,213 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8be22ac0f4cc7a17874b9d10ba204518 2024-11-26T21:49:53,213 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,ggg,1732657792023.8be22ac0f4cc7a17874b9d10ba204518.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:53,213 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/334949367d92cc18a393737b69358f37 2024-11-26T21:49:53,213 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for 8be22ac0f4cc7a17874b9d10ba204518 2024-11-26T21:49:53,213 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for 8be22ac0f4cc7a17874b9d10ba204518 2024-11-26T21:49:53,214 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/334949367d92cc18a393737b69358f37 2024-11-26T21:49:53,215 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=34e003c01eb3f0a2e10e33152402164c, regionState=OPEN, openSeqNum=2, regionLocation=828b3f090c91,42273,1732657789877 2024-11-26T21:49:53,215 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 334949367d92cc18a393737b69358f37 2024-11-26T21:49:53,215 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 334949367d92cc18a393737b69358f37 2024-11-26T21:49:53,216 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,ttt,1732657792023.1ef98738dc2598149d89e210ccdb4e6f. 2024-11-26T21:49:53,216 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => 1ef98738dc2598149d89e210ccdb4e6f, NAME => 'test,ttt,1732657792023.1ef98738dc2598149d89e210ccdb4e6f.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-26T21:49:53,217 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1ef98738dc2598149d89e210ccdb4e6f 2024-11-26T21:49:53,217 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,ttt,1732657792023.1ef98738dc2598149d89e210ccdb4e6f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:53,217 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for 1ef98738dc2598149d89e210ccdb4e6f 2024-11-26T21:49:53,217 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for 1ef98738dc2598149d89e210ccdb4e6f 2024-11-26T21:49:53,218 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 334949367d92cc18a393737b69358f37 2024-11-26T21:49:53,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 34e003c01eb3f0a2e10e33152402164c, server=828b3f090c91,42273,1732657789877 because future has completed 2024-11-26T21:49:53,219 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=651cf53d9624e6cc98e46d6578c71496, regionState=OPEN, openSeqNum=2, regionLocation=828b3f090c91,46545,1732657789755 2024-11-26T21:49:53,221 INFO [StoreOpener-8be22ac0f4cc7a17874b9d10ba204518-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8be22ac0f4cc7a17874b9d10ba204518 2024-11-26T21:49:53,221 INFO [StoreOpener-1ef98738dc2598149d89e210ccdb4e6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1ef98738dc2598149d89e210ccdb4e6f 2024-11-26T21:49:53,222 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,46545,1732657789755 because future has completed 2024-11-26T21:49:53,222 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/334949367d92cc18a393737b69358f37/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T21:49:53,223 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 334949367d92cc18a393737b69358f37; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59894306, jitterRate=-0.10750529170036316}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T21:49:53,223 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 334949367d92cc18a393737b69358f37 2024-11-26T21:49:53,224 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 334949367d92cc18a393737b69358f37: Running coprocessor pre-open hook at 1732657793203Writing region info on filesystem at 1732657793203Initializing all the Stores at 1732657793204 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657793204Cleaning up temporary data from old regions at 1732657793215 (+11 ms)Running coprocessor post-open hooks at 1732657793224 (+9 ms)Region opened successfully at 1732657793224 2024-11-26T21:49:53,224 INFO [StoreOpener-8be22ac0f4cc7a17874b9d10ba204518-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8be22ac0f4cc7a17874b9d10ba204518 columnFamilyName family 2024-11-26T21:49:53,224 INFO [StoreOpener-1ef98738dc2598149d89e210ccdb4e6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1ef98738dc2598149d89e210ccdb4e6f columnFamilyName family 2024-11-26T21:49:53,224 DEBUG [StoreOpener-8be22ac0f4cc7a17874b9d10ba204518-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:53,224 DEBUG [StoreOpener-1ef98738dc2598149d89e210ccdb4e6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:53,225 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1732657792023.334949367d92cc18a393737b69358f37., pid=32, masterSystemTime=1732657793095 2024-11-26T21:49:53,225 INFO [StoreOpener-8be22ac0f4cc7a17874b9d10ba204518-1 {}] regionserver.HStore(327): Store=8be22ac0f4cc7a17874b9d10ba204518/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:49:53,225 INFO [StoreOpener-1ef98738dc2598149d89e210ccdb4e6f-1 {}] regionserver.HStore(327): Store=1ef98738dc2598149d89e210ccdb4e6f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:49:53,225 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for 1ef98738dc2598149d89e210ccdb4e6f 2024-11-26T21:49:53,225 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for 8be22ac0f4cc7a17874b9d10ba204518 2024-11-26T21:49:53,227 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/8be22ac0f4cc7a17874b9d10ba204518 2024-11-26T21:49:53,227 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/1ef98738dc2598149d89e210ccdb4e6f 2024-11-26T21:49:53,227 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/1ef98738dc2598149d89e210ccdb4e6f 2024-11-26T21:49:53,228 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/8be22ac0f4cc7a17874b9d10ba204518 2024-11-26T21:49:53,228 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for 1ef98738dc2598149d89e210ccdb4e6f 2024-11-26T21:49:53,228 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for 1ef98738dc2598149d89e210ccdb4e6f 2024-11-26T21:49:53,228 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for 8be22ac0f4cc7a17874b9d10ba204518 2024-11-26T21:49:53,228 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for 8be22ac0f4cc7a17874b9d10ba204518 2024-11-26T21:49:53,230 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1732657792023.334949367d92cc18a393737b69358f37. 2024-11-26T21:49:53,230 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,kkk,1732657792023.334949367d92cc18a393737b69358f37. 2024-11-26T21:49:53,230 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=7 2024-11-26T21:49:53,230 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 34e003c01eb3f0a2e10e33152402164c, server=828b3f090c91,42273,1732657789877 in 192 msec 2024-11-26T21:49:53,230 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,vvv,1732657792023.21af909e3b8cf553daecee4e59c0898b. 2024-11-26T21:49:53,230 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 21af909e3b8cf553daecee4e59c0898b, NAME => 'test,vvv,1732657792023.21af909e3b8cf553daecee4e59c0898b.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-26T21:49:53,231 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 21af909e3b8cf553daecee4e59c0898b 2024-11-26T21:49:53,231 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,vvv,1732657792023.21af909e3b8cf553daecee4e59c0898b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:53,231 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 21af909e3b8cf553daecee4e59c0898b 2024-11-26T21:49:53,231 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 21af909e3b8cf553daecee4e59c0898b 2024-11-26T21:49:53,232 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for 1ef98738dc2598149d89e210ccdb4e6f 2024-11-26T21:49:53,233 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for 8be22ac0f4cc7a17874b9d10ba204518 2024-11-26T21:49:53,235 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=34e003c01eb3f0a2e10e33152402164c, ASSIGN in 467 msec 2024-11-26T21:49:53,237 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=334949367d92cc18a393737b69358f37, regionState=OPEN, openSeqNum=2, regionLocation=828b3f090c91,46545,1732657789755 2024-11-26T21:49:53,240 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/8be22ac0f4cc7a17874b9d10ba204518/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T21:49:53,240 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/1ef98738dc2598149d89e210ccdb4e6f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T21:49:53,241 INFO [StoreOpener-21af909e3b8cf553daecee4e59c0898b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 21af909e3b8cf553daecee4e59c0898b 2024-11-26T21:49:53,241 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44203 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=828b3f090c91,46545,1732657789755, table=test, region=651cf53d9624e6cc98e46d6578c71496. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T21:49:53,241 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44203 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=828b3f090c91,46545,1732657789755, table=test, region=334949367d92cc18a393737b69358f37. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T21:49:53,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 334949367d92cc18a393737b69358f37, server=828b3f090c91,46545,1732657789755 because future has completed 2024-11-26T21:49:53,244 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened 8be22ac0f4cc7a17874b9d10ba204518; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60306001, jitterRate=-0.10137055814266205}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T21:49:53,244 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened 1ef98738dc2598149d89e210ccdb4e6f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66882098, jitterRate=-0.0033790767192840576}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T21:49:53,244 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8be22ac0f4cc7a17874b9d10ba204518 2024-11-26T21:49:53,244 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1ef98738dc2598149d89e210ccdb4e6f 2024-11-26T21:49:53,244 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for 8be22ac0f4cc7a17874b9d10ba204518: Running coprocessor pre-open hook at 1732657793213Writing region info on filesystem at 1732657793213Initializing all the Stores at 1732657793215 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657793215Cleaning up temporary data from old regions at 1732657793228 (+13 ms)Running coprocessor post-open hooks at 1732657793244 (+16 ms)Region opened successfully at 1732657793244 2024-11-26T21:49:53,244 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for 1ef98738dc2598149d89e210ccdb4e6f: Running coprocessor pre-open hook at 1732657793217Writing region info on filesystem at 1732657793217Initializing all the Stores at 1732657793219 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657793219Cleaning up temporary data from old regions at 1732657793228 (+9 ms)Running coprocessor post-open hooks at 1732657793244 (+16 ms)Region opened successfully at 1732657793244 2024-11-26T21:49:53,246 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1732657792023.8be22ac0f4cc7a17874b9d10ba204518., pid=48, masterSystemTime=1732657793146 2024-11-26T21:49:53,246 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1732657792023.1ef98738dc2598149d89e210ccdb4e6f., pid=50, masterSystemTime=1732657793190 2024-11-26T21:49:53,246 INFO [StoreOpener-21af909e3b8cf553daecee4e59c0898b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 21af909e3b8cf553daecee4e59c0898b columnFamilyName family 2024-11-26T21:49:53,246 DEBUG [StoreOpener-21af909e3b8cf553daecee4e59c0898b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:53,247 INFO [StoreOpener-21af909e3b8cf553daecee4e59c0898b-1 {}] regionserver.HStore(327): Store=21af909e3b8cf553daecee4e59c0898b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:49:53,248 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 21af909e3b8cf553daecee4e59c0898b 2024-11-26T21:49:53,249 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/21af909e3b8cf553daecee4e59c0898b 2024-11-26T21:49:53,249 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/21af909e3b8cf553daecee4e59c0898b 2024-11-26T21:49:53,250 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 21af909e3b8cf553daecee4e59c0898b 2024-11-26T21:49:53,250 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 21af909e3b8cf553daecee4e59c0898b 2024-11-26T21:49:53,252 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 21af909e3b8cf553daecee4e59c0898b 2024-11-26T21:49:53,252 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1732657792023.1ef98738dc2598149d89e210ccdb4e6f. 2024-11-26T21:49:53,252 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,ttt,1732657792023.1ef98738dc2598149d89e210ccdb4e6f. 2024-11-26T21:49:53,253 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,zzz,1732657792023.1062179709b27df36edcb7f6b054a4a6. 2024-11-26T21:49:53,253 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => 1062179709b27df36edcb7f6b054a4a6, NAME => 'test,zzz,1732657792023.1062179709b27df36edcb7f6b054a4a6.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-26T21:49:53,253 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1062179709b27df36edcb7f6b054a4a6 2024-11-26T21:49:53,253 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=9 2024-11-26T21:49:53,253 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,zzz,1732657792023.1062179709b27df36edcb7f6b054a4a6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:53,253 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,46545,1732657789755 in 284 msec 2024-11-26T21:49:53,253 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for 1062179709b27df36edcb7f6b054a4a6 2024-11-26T21:49:53,253 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for 1062179709b27df36edcb7f6b054a4a6 2024-11-26T21:49:53,256 INFO [StoreOpener-1062179709b27df36edcb7f6b054a4a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1062179709b27df36edcb7f6b054a4a6 2024-11-26T21:49:53,256 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/21af909e3b8cf553daecee4e59c0898b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T21:49:53,256 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1732657792023.8be22ac0f4cc7a17874b9d10ba204518. 2024-11-26T21:49:53,256 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,ggg,1732657792023.8be22ac0f4cc7a17874b9d10ba204518. 2024-11-26T21:49:53,256 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,rrr,1732657792023.30cf2d9c6c6cf36bd61bb35f51df8f42. 2024-11-26T21:49:53,257 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 30cf2d9c6c6cf36bd61bb35f51df8f42, NAME => 'test,rrr,1732657792023.30cf2d9c6c6cf36bd61bb35f51df8f42.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-26T21:49:53,257 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 21af909e3b8cf553daecee4e59c0898b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60948739, jitterRate=-0.0917930155992508}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T21:49:53,257 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 21af909e3b8cf553daecee4e59c0898b 2024-11-26T21:49:53,257 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 21af909e3b8cf553daecee4e59c0898b: Running coprocessor pre-open hook at 1732657793232Writing region info on filesystem at 1732657793232Initializing all the Stores at 1732657793234 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657793234Cleaning up temporary data from old regions at 1732657793250 (+16 ms)Running coprocessor post-open hooks at 1732657793257 (+7 ms)Region opened successfully at 1732657793257 2024-11-26T21:49:53,257 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN in 490 msec 2024-11-26T21:49:53,257 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 30cf2d9c6c6cf36bd61bb35f51df8f42 2024-11-26T21:49:53,258 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,rrr,1732657792023.30cf2d9c6c6cf36bd61bb35f51df8f42.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:53,258 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 30cf2d9c6c6cf36bd61bb35f51df8f42 2024-11-26T21:49:53,258 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 30cf2d9c6c6cf36bd61bb35f51df8f42 2024-11-26T21:49:53,259 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1732657792023.21af909e3b8cf553daecee4e59c0898b., pid=37, masterSystemTime=1732657793095 2024-11-26T21:49:53,259 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=8be22ac0f4cc7a17874b9d10ba204518, regionState=OPEN, openSeqNum=2, regionLocation=828b3f090c91,42273,1732657789877 2024-11-26T21:49:53,261 INFO [StoreOpener-1062179709b27df36edcb7f6b054a4a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1062179709b27df36edcb7f6b054a4a6 columnFamilyName family 2024-11-26T21:49:53,261 INFO [StoreOpener-30cf2d9c6c6cf36bd61bb35f51df8f42-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 30cf2d9c6c6cf36bd61bb35f51df8f42 2024-11-26T21:49:53,261 DEBUG [StoreOpener-1062179709b27df36edcb7f6b054a4a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:53,262 INFO [StoreOpener-1062179709b27df36edcb7f6b054a4a6-1 {}] regionserver.HStore(327): Store=1062179709b27df36edcb7f6b054a4a6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:49:53,263 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1732657792023.21af909e3b8cf553daecee4e59c0898b. 2024-11-26T21:49:53,263 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8be22ac0f4cc7a17874b9d10ba204518, server=828b3f090c91,42273,1732657789877 because future has completed 2024-11-26T21:49:53,263 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for 1062179709b27df36edcb7f6b054a4a6 2024-11-26T21:49:53,263 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,vvv,1732657792023.21af909e3b8cf553daecee4e59c0898b. 2024-11-26T21:49:53,264 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,sss,1732657792023.2b8b8290cc9d27b8d226e5dc2b7d770d. 2024-11-26T21:49:53,264 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 2b8b8290cc9d27b8d226e5dc2b7d770d, NAME => 'test,sss,1732657792023.2b8b8290cc9d27b8d226e5dc2b7d770d.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-26T21:49:53,264 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2b8b8290cc9d27b8d226e5dc2b7d770d 2024-11-26T21:49:53,264 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,sss,1732657792023.2b8b8290cc9d27b8d226e5dc2b7d770d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:53,264 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 2b8b8290cc9d27b8d226e5dc2b7d770d 2024-11-26T21:49:53,264 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 2b8b8290cc9d27b8d226e5dc2b7d770d 2024-11-26T21:49:53,265 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/1062179709b27df36edcb7f6b054a4a6 2024-11-26T21:49:53,265 INFO [StoreOpener-30cf2d9c6c6cf36bd61bb35f51df8f42-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 30cf2d9c6c6cf36bd61bb35f51df8f42 columnFamilyName family 2024-11-26T21:49:53,265 DEBUG [StoreOpener-30cf2d9c6c6cf36bd61bb35f51df8f42-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:53,265 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/1062179709b27df36edcb7f6b054a4a6 2024-11-26T21:49:53,266 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=21af909e3b8cf553daecee4e59c0898b, regionState=OPEN, openSeqNum=2, regionLocation=828b3f090c91,46545,1732657789755 2024-11-26T21:49:53,266 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for 1062179709b27df36edcb7f6b054a4a6 2024-11-26T21:49:53,266 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for 1062179709b27df36edcb7f6b054a4a6 2024-11-26T21:49:53,266 INFO [StoreOpener-2b8b8290cc9d27b8d226e5dc2b7d770d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2b8b8290cc9d27b8d226e5dc2b7d770d 2024-11-26T21:49:53,267 INFO [StoreOpener-30cf2d9c6c6cf36bd61bb35f51df8f42-1 {}] regionserver.HStore(327): Store=30cf2d9c6c6cf36bd61bb35f51df8f42/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:49:53,268 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 30cf2d9c6c6cf36bd61bb35f51df8f42 2024-11-26T21:49:53,269 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/30cf2d9c6c6cf36bd61bb35f51df8f42 2024-11-26T21:49:53,270 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/30cf2d9c6c6cf36bd61bb35f51df8f42 2024-11-26T21:49:53,270 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21af909e3b8cf553daecee4e59c0898b, server=828b3f090c91,46545,1732657789755 because future has completed 2024-11-26T21:49:53,271 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 30cf2d9c6c6cf36bd61bb35f51df8f42 2024-11-26T21:49:53,271 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for 1062179709b27df36edcb7f6b054a4a6 2024-11-26T21:49:53,271 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 30cf2d9c6c6cf36bd61bb35f51df8f42 2024-11-26T21:49:53,271 INFO [StoreOpener-2b8b8290cc9d27b8d226e5dc2b7d770d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b8b8290cc9d27b8d226e5dc2b7d770d columnFamilyName family 2024-11-26T21:49:53,272 DEBUG [StoreOpener-2b8b8290cc9d27b8d226e5dc2b7d770d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:53,272 INFO [StoreOpener-2b8b8290cc9d27b8d226e5dc2b7d770d-1 {}] regionserver.HStore(327): Store=2b8b8290cc9d27b8d226e5dc2b7d770d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:49:53,273 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 2b8b8290cc9d27b8d226e5dc2b7d770d 2024-11-26T21:49:53,274 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 30cf2d9c6c6cf36bd61bb35f51df8f42 2024-11-26T21:49:53,274 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/2b8b8290cc9d27b8d226e5dc2b7d770d 2024-11-26T21:49:53,275 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/2b8b8290cc9d27b8d226e5dc2b7d770d 2024-11-26T21:49:53,276 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 2b8b8290cc9d27b8d226e5dc2b7d770d 2024-11-26T21:49:53,276 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 2b8b8290cc9d27b8d226e5dc2b7d770d 2024-11-26T21:49:53,276 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/1062179709b27df36edcb7f6b054a4a6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T21:49:53,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=11 2024-11-26T21:49:53,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 8be22ac0f4cc7a17874b9d10ba204518, server=828b3f090c91,42273,1732657789877 in 226 msec 2024-11-26T21:49:53,278 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened 1062179709b27df36edcb7f6b054a4a6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64551784, jitterRate=-0.038103461265563965}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T21:49:53,279 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/30cf2d9c6c6cf36bd61bb35f51df8f42/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T21:49:53,279 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1062179709b27df36edcb7f6b054a4a6 2024-11-26T21:49:53,279 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for 1062179709b27df36edcb7f6b054a4a6: Running coprocessor pre-open hook at 1732657793254Writing region info on filesystem at 1732657793254Initializing all the Stores at 1732657793255 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657793255Cleaning up temporary data from old regions at 1732657793266 (+11 ms)Running coprocessor post-open hooks at 1732657793279 (+13 ms)Region opened successfully at 1732657793279 2024-11-26T21:49:53,280 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 2b8b8290cc9d27b8d226e5dc2b7d770d 2024-11-26T21:49:53,280 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 30cf2d9c6c6cf36bd61bb35f51df8f42; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65359294, jitterRate=-0.026070624589920044}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T21:49:53,280 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 30cf2d9c6c6cf36bd61bb35f51df8f42 2024-11-26T21:49:53,280 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 30cf2d9c6c6cf36bd61bb35f51df8f42: Running coprocessor pre-open hook at 1732657793258Writing region info on filesystem at 1732657793258Initializing all the Stores at 1732657793260 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657793260Cleaning up temporary data from old regions at 1732657793271 (+11 ms)Running coprocessor post-open hooks at 1732657793280 (+9 ms)Region opened successfully at 1732657793280 2024-11-26T21:49:53,280 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1732657792023.1062179709b27df36edcb7f6b054a4a6., pid=47, masterSystemTime=1732657793190 2024-11-26T21:49:53,281 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1732657792023.30cf2d9c6c6cf36bd61bb35f51df8f42., pid=39, masterSystemTime=1732657793146 2024-11-26T21:49:53,283 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8be22ac0f4cc7a17874b9d10ba204518, ASSIGN in 514 msec 2024-11-26T21:49:53,283 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1732657792023.1062179709b27df36edcb7f6b054a4a6. 2024-11-26T21:49:53,283 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,zzz,1732657792023.1062179709b27df36edcb7f6b054a4a6. 2024-11-26T21:49:53,284 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,ddd,1732657792023.7a7c374815f888db179beadaf2489ad7. 2024-11-26T21:49:53,284 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => 7a7c374815f888db179beadaf2489ad7, NAME => 'test,ddd,1732657792023.7a7c374815f888db179beadaf2489ad7.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-26T21:49:53,284 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7a7c374815f888db179beadaf2489ad7 2024-11-26T21:49:53,284 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,ddd,1732657792023.7a7c374815f888db179beadaf2489ad7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:53,285 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for 7a7c374815f888db179beadaf2489ad7 2024-11-26T21:49:53,285 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for 7a7c374815f888db179beadaf2489ad7 2024-11-26T21:49:53,285 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/2b8b8290cc9d27b8d226e5dc2b7d770d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T21:49:53,285 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1732657792023.30cf2d9c6c6cf36bd61bb35f51df8f42. 2024-11-26T21:49:53,285 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,rrr,1732657792023.30cf2d9c6c6cf36bd61bb35f51df8f42. 2024-11-26T21:49:53,285 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,ppp,1732657792023.d1c5d2bc20a7b33de1c8b5a53f7fee57. 2024-11-26T21:49:53,285 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=1062179709b27df36edcb7f6b054a4a6, regionState=OPEN, openSeqNum=2, regionLocation=828b3f090c91,40359,1732657789839 2024-11-26T21:49:53,285 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => d1c5d2bc20a7b33de1c8b5a53f7fee57, NAME => 'test,ppp,1732657792023.d1c5d2bc20a7b33de1c8b5a53f7fee57.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-26T21:49:53,286 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d1c5d2bc20a7b33de1c8b5a53f7fee57 2024-11-26T21:49:53,286 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,ppp,1732657792023.d1c5d2bc20a7b33de1c8b5a53f7fee57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:53,286 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for d1c5d2bc20a7b33de1c8b5a53f7fee57 2024-11-26T21:49:53,286 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 2b8b8290cc9d27b8d226e5dc2b7d770d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74039464, jitterRate=0.10327398777008057}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T21:49:53,286 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for d1c5d2bc20a7b33de1c8b5a53f7fee57 2024-11-26T21:49:53,286 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2b8b8290cc9d27b8d226e5dc2b7d770d 2024-11-26T21:49:53,286 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 2b8b8290cc9d27b8d226e5dc2b7d770d: Running coprocessor pre-open hook at 1732657793265Writing region info on filesystem at 1732657793265Initializing all the Stores at 1732657793266 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657793266Cleaning up temporary data from old regions at 1732657793276 (+10 ms)Running coprocessor post-open hooks at 1732657793286 (+10 ms)Region opened successfully at 1732657793286 2024-11-26T21:49:53,288 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1732657792023.2b8b8290cc9d27b8d226e5dc2b7d770d., pid=36, masterSystemTime=1732657793095 2024-11-26T21:49:53,289 INFO [StoreOpener-7a7c374815f888db179beadaf2489ad7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7a7c374815f888db179beadaf2489ad7 2024-11-26T21:49:53,290 INFO [StoreOpener-d1c5d2bc20a7b33de1c8b5a53f7fee57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d1c5d2bc20a7b33de1c8b5a53f7fee57 2024-11-26T21:49:53,290 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1062179709b27df36edcb7f6b054a4a6, server=828b3f090c91,40359,1732657789839 because future has completed 2024-11-26T21:49:53,293 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=30cf2d9c6c6cf36bd61bb35f51df8f42, regionState=OPEN, openSeqNum=2, regionLocation=828b3f090c91,42273,1732657789877 2024-11-26T21:49:53,293 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1732657792023.2b8b8290cc9d27b8d226e5dc2b7d770d. 2024-11-26T21:49:53,293 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,sss,1732657792023.2b8b8290cc9d27b8d226e5dc2b7d770d. 2024-11-26T21:49:53,293 INFO [StoreOpener-d1c5d2bc20a7b33de1c8b5a53f7fee57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1c5d2bc20a7b33de1c8b5a53f7fee57 columnFamilyName family 2024-11-26T21:49:53,293 DEBUG [StoreOpener-d1c5d2bc20a7b33de1c8b5a53f7fee57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:53,293 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,yyy,1732657792023.1e7efa0e8c09fd1f0e1a765d7fc38181. 2024-11-26T21:49:53,293 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 1e7efa0e8c09fd1f0e1a765d7fc38181, NAME => 'test,yyy,1732657792023.1e7efa0e8c09fd1f0e1a765d7fc38181.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-26T21:49:53,293 INFO [StoreOpener-7a7c374815f888db179beadaf2489ad7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7a7c374815f888db179beadaf2489ad7 columnFamilyName family 2024-11-26T21:49:53,294 DEBUG [StoreOpener-7a7c374815f888db179beadaf2489ad7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:53,294 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1e7efa0e8c09fd1f0e1a765d7fc38181 2024-11-26T21:49:53,294 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,yyy,1732657792023.1e7efa0e8c09fd1f0e1a765d7fc38181.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:53,294 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 1e7efa0e8c09fd1f0e1a765d7fc38181 2024-11-26T21:49:53,294 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 1e7efa0e8c09fd1f0e1a765d7fc38181 2024-11-26T21:49:53,294 INFO [StoreOpener-7a7c374815f888db179beadaf2489ad7-1 {}] regionserver.HStore(327): Store=7a7c374815f888db179beadaf2489ad7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:49:53,294 INFO [StoreOpener-d1c5d2bc20a7b33de1c8b5a53f7fee57-1 {}] regionserver.HStore(327): Store=d1c5d2bc20a7b33de1c8b5a53f7fee57/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:49:53,294 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for d1c5d2bc20a7b33de1c8b5a53f7fee57 2024-11-26T21:49:53,295 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for 7a7c374815f888db179beadaf2489ad7 2024-11-26T21:49:53,296 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/d1c5d2bc20a7b33de1c8b5a53f7fee57 2024-11-26T21:49:53,296 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/7a7c374815f888db179beadaf2489ad7 2024-11-26T21:49:53,297 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 30cf2d9c6c6cf36bd61bb35f51df8f42, server=828b3f090c91,42273,1732657789877 because future has completed 2024-11-26T21:49:53,297 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/7a7c374815f888db179beadaf2489ad7 2024-11-26T21:49:53,297 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/d1c5d2bc20a7b33de1c8b5a53f7fee57 2024-11-26T21:49:53,298 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for 7a7c374815f888db179beadaf2489ad7 2024-11-26T21:49:53,298 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for 7a7c374815f888db179beadaf2489ad7 2024-11-26T21:49:53,299 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for d1c5d2bc20a7b33de1c8b5a53f7fee57 2024-11-26T21:49:53,299 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for d1c5d2bc20a7b33de1c8b5a53f7fee57 2024-11-26T21:49:53,302 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for 7a7c374815f888db179beadaf2489ad7 2024-11-26T21:49:53,303 INFO [StoreOpener-1e7efa0e8c09fd1f0e1a765d7fc38181-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1e7efa0e8c09fd1f0e1a765d7fc38181 2024-11-26T21:49:53,303 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for d1c5d2bc20a7b33de1c8b5a53f7fee57 2024-11-26T21:49:53,306 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=22 2024-11-26T21:49:53,306 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure 30cf2d9c6c6cf36bd61bb35f51df8f42, server=828b3f090c91,42273,1732657789877 in 308 msec 2024-11-26T21:49:53,307 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/7a7c374815f888db179beadaf2489ad7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T21:49:53,306 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/d1c5d2bc20a7b33de1c8b5a53f7fee57/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T21:49:53,306 INFO [StoreOpener-1e7efa0e8c09fd1f0e1a765d7fc38181-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e7efa0e8c09fd1f0e1a765d7fc38181 columnFamilyName family 2024-11-26T21:49:53,307 DEBUG [StoreOpener-1e7efa0e8c09fd1f0e1a765d7fc38181-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:53,307 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened 7a7c374815f888db179beadaf2489ad7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60430566, jitterRate=-0.09951439499855042}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T21:49:53,308 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7a7c374815f888db179beadaf2489ad7 2024-11-26T21:49:53,308 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for 7a7c374815f888db179beadaf2489ad7: Running coprocessor pre-open hook at 1732657793285Writing region info on filesystem at 1732657793285Initializing all the Stores at 1732657793287 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657793287Cleaning up temporary data from old regions at 1732657793298 (+11 ms)Running coprocessor post-open hooks at 1732657793308 (+10 ms)Region opened successfully at 1732657793308 2024-11-26T21:49:53,308 INFO [StoreOpener-1e7efa0e8c09fd1f0e1a765d7fc38181-1 {}] regionserver.HStore(327): Store=1e7efa0e8c09fd1f0e1a765d7fc38181/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:49:53,308 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 1e7efa0e8c09fd1f0e1a765d7fc38181 2024-11-26T21:49:53,309 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened d1c5d2bc20a7b33de1c8b5a53f7fee57; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65700328, jitterRate=-0.020988821983337402}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T21:49:53,309 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1732657792023.7a7c374815f888db179beadaf2489ad7., pid=52, masterSystemTime=1732657793190 2024-11-26T21:49:53,309 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d1c5d2bc20a7b33de1c8b5a53f7fee57 2024-11-26T21:49:53,309 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for d1c5d2bc20a7b33de1c8b5a53f7fee57: Running coprocessor pre-open hook at 1732657793286Writing region info on filesystem at 1732657793286Initializing all the Stores at 1732657793288 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657793288Cleaning up temporary data from old regions at 1732657793299 (+11 ms)Running coprocessor post-open hooks at 1732657793309 (+10 ms)Region opened successfully at 1732657793309 2024-11-26T21:49:53,309 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/1e7efa0e8c09fd1f0e1a765d7fc38181 2024-11-26T21:49:53,310 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=30cf2d9c6c6cf36bd61bb35f51df8f42, ASSIGN in 543 msec 2024-11-26T21:49:53,311 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1732657792023.d1c5d2bc20a7b33de1c8b5a53f7fee57., pid=40, masterSystemTime=1732657793146 2024-11-26T21:49:53,311 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/1e7efa0e8c09fd1f0e1a765d7fc38181 2024-11-26T21:49:53,312 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=2b8b8290cc9d27b8d226e5dc2b7d770d, regionState=OPEN, openSeqNum=2, regionLocation=828b3f090c91,46545,1732657789755 2024-11-26T21:49:53,312 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1732657792023.7a7c374815f888db179beadaf2489ad7. 2024-11-26T21:49:53,312 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,ddd,1732657792023.7a7c374815f888db179beadaf2489ad7. 2024-11-26T21:49:53,312 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,bbb,1732657792023.36699c847bc0e6c6b06fd8533d3c9edb. 2024-11-26T21:49:53,313 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => 36699c847bc0e6c6b06fd8533d3c9edb, NAME => 'test,bbb,1732657792023.36699c847bc0e6c6b06fd8533d3c9edb.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-26T21:49:53,313 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 36699c847bc0e6c6b06fd8533d3c9edb 2024-11-26T21:49:53,313 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,bbb,1732657792023.36699c847bc0e6c6b06fd8533d3c9edb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:53,313 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for 36699c847bc0e6c6b06fd8533d3c9edb 2024-11-26T21:49:53,313 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for 36699c847bc0e6c6b06fd8533d3c9edb 2024-11-26T21:49:53,314 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 1e7efa0e8c09fd1f0e1a765d7fc38181 2024-11-26T21:49:53,315 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 1e7efa0e8c09fd1f0e1a765d7fc38181 2024-11-26T21:49:53,315 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1732657792023.d1c5d2bc20a7b33de1c8b5a53f7fee57. 2024-11-26T21:49:53,315 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,ppp,1732657792023.d1c5d2bc20a7b33de1c8b5a53f7fee57. 2024-11-26T21:49:53,315 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,jjj,1732657792023.cdd2d12bbd55678c7f638b4ef0fde3da. 2024-11-26T21:49:53,315 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => cdd2d12bbd55678c7f638b4ef0fde3da, NAME => 'test,jjj,1732657792023.cdd2d12bbd55678c7f638b4ef0fde3da.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-26T21:49:53,316 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test cdd2d12bbd55678c7f638b4ef0fde3da 2024-11-26T21:49:53,316 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,jjj,1732657792023.cdd2d12bbd55678c7f638b4ef0fde3da.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:53,316 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for cdd2d12bbd55678c7f638b4ef0fde3da 2024-11-26T21:49:53,316 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for cdd2d12bbd55678c7f638b4ef0fde3da 2024-11-26T21:49:53,316 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b8b8290cc9d27b8d226e5dc2b7d770d, server=828b3f090c91,46545,1732657789755 because future has completed 2024-11-26T21:49:53,318 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=d1c5d2bc20a7b33de1c8b5a53f7fee57, regionState=OPEN, openSeqNum=2, regionLocation=828b3f090c91,42273,1732657789877 2024-11-26T21:49:53,318 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 1e7efa0e8c09fd1f0e1a765d7fc38181 2024-11-26T21:49:53,321 INFO [StoreOpener-cdd2d12bbd55678c7f638b4ef0fde3da-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region cdd2d12bbd55678c7f638b4ef0fde3da 2024-11-26T21:49:53,322 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1c5d2bc20a7b33de1c8b5a53f7fee57, server=828b3f090c91,42273,1732657789877 because future has completed 2024-11-26T21:49:53,322 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/1e7efa0e8c09fd1f0e1a765d7fc38181/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T21:49:53,322 INFO [StoreOpener-36699c847bc0e6c6b06fd8533d3c9edb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 36699c847bc0e6c6b06fd8533d3c9edb 2024-11-26T21:49:53,323 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 1e7efa0e8c09fd1f0e1a765d7fc38181; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74057700, jitterRate=0.10354572534561157}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T21:49:53,323 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1e7efa0e8c09fd1f0e1a765d7fc38181 2024-11-26T21:49:53,323 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 1e7efa0e8c09fd1f0e1a765d7fc38181: Running coprocessor pre-open hook at 1732657793294Writing region info on filesystem at 1732657793294Initializing all the Stores at 1732657793297 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657793297Cleaning up temporary data from old regions at 1732657793315 (+18 ms)Running coprocessor post-open hooks at 1732657793323 (+8 ms)Region opened successfully at 1732657793323 2024-11-26T21:49:53,325 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1732657792023.1e7efa0e8c09fd1f0e1a765d7fc38181., pid=31, masterSystemTime=1732657793095 2024-11-26T21:49:53,325 INFO [StoreOpener-cdd2d12bbd55678c7f638b4ef0fde3da-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cdd2d12bbd55678c7f638b4ef0fde3da columnFamilyName family 2024-11-26T21:49:53,325 DEBUG [StoreOpener-cdd2d12bbd55678c7f638b4ef0fde3da-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:53,325 INFO [StoreOpener-36699c847bc0e6c6b06fd8533d3c9edb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 36699c847bc0e6c6b06fd8533d3c9edb columnFamilyName family 2024-11-26T21:49:53,325 DEBUG [StoreOpener-36699c847bc0e6c6b06fd8533d3c9edb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:53,326 INFO [StoreOpener-cdd2d12bbd55678c7f638b4ef0fde3da-1 {}] regionserver.HStore(327): Store=cdd2d12bbd55678c7f638b4ef0fde3da/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:49:53,326 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for cdd2d12bbd55678c7f638b4ef0fde3da 2024-11-26T21:49:53,327 INFO [StoreOpener-36699c847bc0e6c6b06fd8533d3c9edb-1 {}] regionserver.HStore(327): Store=36699c847bc0e6c6b06fd8533d3c9edb/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:49:53,327 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for 36699c847bc0e6c6b06fd8533d3c9edb 2024-11-26T21:49:53,327 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/cdd2d12bbd55678c7f638b4ef0fde3da 2024-11-26T21:49:53,328 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/cdd2d12bbd55678c7f638b4ef0fde3da 2024-11-26T21:49:53,328 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/36699c847bc0e6c6b06fd8533d3c9edb 2024-11-26T21:49:53,329 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for cdd2d12bbd55678c7f638b4ef0fde3da 2024-11-26T21:49:53,329 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for cdd2d12bbd55678c7f638b4ef0fde3da 2024-11-26T21:49:53,329 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/36699c847bc0e6c6b06fd8533d3c9edb 2024-11-26T21:49:53,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=23 2024-11-26T21:49:53,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 2b8b8290cc9d27b8d226e5dc2b7d770d, server=828b3f090c91,46545,1732657789755 in 348 msec 2024-11-26T21:49:53,330 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for 36699c847bc0e6c6b06fd8533d3c9edb 2024-11-26T21:49:53,330 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for 36699c847bc0e6c6b06fd8533d3c9edb 2024-11-26T21:49:53,330 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1732657792023.1e7efa0e8c09fd1f0e1a765d7fc38181. 2024-11-26T21:49:53,330 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,yyy,1732657792023.1e7efa0e8c09fd1f0e1a765d7fc38181. 2024-11-26T21:49:53,331 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=1e7efa0e8c09fd1f0e1a765d7fc38181, regionState=OPEN, openSeqNum=2, regionLocation=828b3f090c91,46545,1732657789755 2024-11-26T21:49:53,331 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for cdd2d12bbd55678c7f638b4ef0fde3da 2024-11-26T21:49:53,332 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for 36699c847bc0e6c6b06fd8533d3c9edb 2024-11-26T21:49:53,334 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e7efa0e8c09fd1f0e1a765d7fc38181, server=828b3f090c91,46545,1732657789755 because future has completed 2024-11-26T21:49:53,337 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/36699c847bc0e6c6b06fd8533d3c9edb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T21:49:53,337 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/cdd2d12bbd55678c7f638b4ef0fde3da/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T21:49:53,337 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN in 570 msec 2024-11-26T21:49:53,338 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44203 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=828b3f090c91,42273,1732657789877, table=test, region=d1c5d2bc20a7b33de1c8b5a53f7fee57. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T21:49:53,338 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened cdd2d12bbd55678c7f638b4ef0fde3da; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66196482, jitterRate=-0.013595551252365112}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T21:49:53,339 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened 36699c847bc0e6c6b06fd8533d3c9edb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61038518, jitterRate=-0.09045520424842834}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T21:49:53,339 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cdd2d12bbd55678c7f638b4ef0fde3da 2024-11-26T21:49:53,339 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 36699c847bc0e6c6b06fd8533d3c9edb 2024-11-26T21:49:53,339 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for 36699c847bc0e6c6b06fd8533d3c9edb: Running coprocessor pre-open hook at 1732657793313Writing region info on filesystem at 1732657793314 (+1 ms)Initializing all the Stores at 1732657793315 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657793315Cleaning up temporary data from old regions at 1732657793330 (+15 ms)Running coprocessor post-open hooks at 1732657793339 (+9 ms)Region opened successfully at 1732657793339 2024-11-26T21:49:53,339 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for cdd2d12bbd55678c7f638b4ef0fde3da: Running coprocessor pre-open hook at 1732657793316Writing region info on filesystem at 1732657793316Initializing all the Stores at 1732657793317 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657793317Cleaning up temporary data from old regions at 1732657793329 (+12 ms)Running coprocessor post-open hooks at 1732657793339 (+10 ms)Region opened successfully at 1732657793339 2024-11-26T21:49:53,341 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1732657792023.cdd2d12bbd55678c7f638b4ef0fde3da., pid=49, masterSystemTime=1732657793146 2024-11-26T21:49:53,341 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44203 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=828b3f090c91,40359,1732657789839, table=test, region=1062179709b27df36edcb7f6b054a4a6. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T21:49:53,342 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1732657792023.36699c847bc0e6c6b06fd8533d3c9edb., pid=56, masterSystemTime=1732657793190 2024-11-26T21:49:53,346 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1732657792023.cdd2d12bbd55678c7f638b4ef0fde3da. 2024-11-26T21:49:53,346 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,jjj,1732657792023.cdd2d12bbd55678c7f638b4ef0fde3da. 2024-11-26T21:49:53,346 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,xxx,1732657792023.0e9187292024abc7067d470edad3ae67. 2024-11-26T21:49:53,346 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 0e9187292024abc7067d470edad3ae67, NAME => 'test,xxx,1732657792023.0e9187292024abc7067d470edad3ae67.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-26T21:49:53,347 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0e9187292024abc7067d470edad3ae67 2024-11-26T21:49:53,347 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,xxx,1732657792023.0e9187292024abc7067d470edad3ae67.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:53,347 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 0e9187292024abc7067d470edad3ae67 2024-11-26T21:49:53,347 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 0e9187292024abc7067d470edad3ae67 2024-11-26T21:49:53,350 INFO [StoreOpener-0e9187292024abc7067d470edad3ae67-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0e9187292024abc7067d470edad3ae67 2024-11-26T21:49:53,350 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=29 2024-11-26T21:49:53,350 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 1e7efa0e8c09fd1f0e1a765d7fc38181, server=828b3f090c91,46545,1732657789755 in 402 msec 2024-11-26T21:49:53,350 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1732657792023.36699c847bc0e6c6b06fd8533d3c9edb. 2024-11-26T21:49:53,350 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,bbb,1732657792023.36699c847bc0e6c6b06fd8533d3c9edb. 2024-11-26T21:49:53,350 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,fff,1732657792023.f0a1ab633a90e6b5f5e9ba8ce6f66010. 2024-11-26T21:49:53,351 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => f0a1ab633a90e6b5f5e9ba8ce6f66010, NAME => 'test,fff,1732657792023.f0a1ab633a90e6b5f5e9ba8ce6f66010.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-26T21:49:53,351 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44203 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=828b3f090c91,46545,1732657789755, table=test, region=334949367d92cc18a393737b69358f37. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T21:49:53,351 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44203 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=828b3f090c91,46545,1732657789755, table=test, region=21af909e3b8cf553daecee4e59c0898b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T21:49:53,351 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f0a1ab633a90e6b5f5e9ba8ce6f66010 2024-11-26T21:49:53,351 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,fff,1732657792023.f0a1ab633a90e6b5f5e9ba8ce6f66010.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:53,351 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for f0a1ab633a90e6b5f5e9ba8ce6f66010 2024-11-26T21:49:53,351 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for f0a1ab633a90e6b5f5e9ba8ce6f66010 2024-11-26T21:49:53,352 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=36699c847bc0e6c6b06fd8533d3c9edb, regionState=OPEN, openSeqNum=2, regionLocation=828b3f090c91,40359,1732657789839 2024-11-26T21:49:53,354 INFO [StoreOpener-0e9187292024abc7067d470edad3ae67-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0e9187292024abc7067d470edad3ae67 columnFamilyName family 2024-11-26T21:49:53,354 INFO [StoreOpener-f0a1ab633a90e6b5f5e9ba8ce6f66010-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f0a1ab633a90e6b5f5e9ba8ce6f66010 2024-11-26T21:49:53,354 DEBUG [StoreOpener-0e9187292024abc7067d470edad3ae67-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:53,356 INFO [StoreOpener-0e9187292024abc7067d470edad3ae67-1 {}] regionserver.HStore(327): Store=0e9187292024abc7067d470edad3ae67/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:49:53,356 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 0e9187292024abc7067d470edad3ae67 2024-11-26T21:49:53,356 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 36699c847bc0e6c6b06fd8533d3c9edb, server=828b3f090c91,40359,1732657789839 because future has completed 2024-11-26T21:49:53,357 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN in 591 msec 2024-11-26T21:49:53,357 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/0e9187292024abc7067d470edad3ae67 2024-11-26T21:49:53,358 INFO [StoreOpener-f0a1ab633a90e6b5f5e9ba8ce6f66010-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f0a1ab633a90e6b5f5e9ba8ce6f66010 columnFamilyName family 2024-11-26T21:49:53,358 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/0e9187292024abc7067d470edad3ae67 2024-11-26T21:49:53,358 DEBUG [StoreOpener-f0a1ab633a90e6b5f5e9ba8ce6f66010-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:53,359 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 0e9187292024abc7067d470edad3ae67 2024-11-26T21:49:53,359 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 0e9187292024abc7067d470edad3ae67 2024-11-26T21:49:53,359 INFO [StoreOpener-f0a1ab633a90e6b5f5e9ba8ce6f66010-1 {}] regionserver.HStore(327): Store=f0a1ab633a90e6b5f5e9ba8ce6f66010/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:49:53,359 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for f0a1ab633a90e6b5f5e9ba8ce6f66010 2024-11-26T21:49:53,360 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/f0a1ab633a90e6b5f5e9ba8ce6f66010 2024-11-26T21:49:53,360 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/f0a1ab633a90e6b5f5e9ba8ce6f66010 2024-11-26T21:49:53,361 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for f0a1ab633a90e6b5f5e9ba8ce6f66010 2024-11-26T21:49:53,361 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for f0a1ab633a90e6b5f5e9ba8ce6f66010 2024-11-26T21:49:53,361 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 0e9187292024abc7067d470edad3ae67 2024-11-26T21:49:53,364 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for f0a1ab633a90e6b5f5e9ba8ce6f66010 2024-11-26T21:49:53,365 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=6 2024-11-26T21:49:53,365 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 36699c847bc0e6c6b06fd8533d3c9edb, server=828b3f090c91,40359,1732657789839 in 272 msec 2024-11-26T21:49:53,366 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/0e9187292024abc7067d470edad3ae67/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T21:49:53,367 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/f0a1ab633a90e6b5f5e9ba8ce6f66010/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T21:49:53,367 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 0e9187292024abc7067d470edad3ae67; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59331649, jitterRate=-0.1158895343542099}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T21:49:53,368 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0e9187292024abc7067d470edad3ae67 2024-11-26T21:49:53,368 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 0e9187292024abc7067d470edad3ae67: Running coprocessor pre-open hook at 1732657793347Writing region info on filesystem at 1732657793347Initializing all the Stores at 1732657793349 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657793349Cleaning up temporary data from old regions at 1732657793359 (+10 ms)Running coprocessor post-open hooks at 1732657793368 (+9 ms)Region opened successfully at 1732657793368 2024-11-26T21:49:53,368 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened f0a1ab633a90e6b5f5e9ba8ce6f66010; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61459551, jitterRate=-0.08418132364749908}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T21:49:53,368 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f0a1ab633a90e6b5f5e9ba8ce6f66010 2024-11-26T21:49:53,368 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for f0a1ab633a90e6b5f5e9ba8ce6f66010: Running coprocessor pre-open hook at 1732657793352Writing region info on filesystem at 1732657793352Initializing all the Stores at 1732657793353 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657793353Cleaning up temporary data from old regions at 1732657793361 (+8 ms)Running coprocessor post-open hooks at 1732657793368 (+7 ms)Region opened successfully at 1732657793368 2024-11-26T21:49:53,369 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1732657792023.0e9187292024abc7067d470edad3ae67., pid=43, masterSystemTime=1732657793146 2024-11-26T21:49:53,369 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1732657792023.f0a1ab633a90e6b5f5e9ba8ce6f66010., pid=51, masterSystemTime=1732657793190 2024-11-26T21:49:53,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=36699c847bc0e6c6b06fd8533d3c9edb, ASSIGN in 602 msec 2024-11-26T21:49:53,371 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=cdd2d12bbd55678c7f638b4ef0fde3da, regionState=OPEN, openSeqNum=2, regionLocation=828b3f090c91,42273,1732657789877 2024-11-26T21:49:53,371 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1732657792023.0e9187292024abc7067d470edad3ae67. 2024-11-26T21:49:53,371 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,xxx,1732657792023.0e9187292024abc7067d470edad3ae67. 2024-11-26T21:49:53,372 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,qqq,1732657792023.996589dbb2419ea602bc5e348cbd05df. 2024-11-26T21:49:53,372 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 996589dbb2419ea602bc5e348cbd05df, NAME => 'test,qqq,1732657792023.996589dbb2419ea602bc5e348cbd05df.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-26T21:49:53,372 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 996589dbb2419ea602bc5e348cbd05df 2024-11-26T21:49:53,372 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,qqq,1732657792023.996589dbb2419ea602bc5e348cbd05df.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:53,373 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 996589dbb2419ea602bc5e348cbd05df 2024-11-26T21:49:53,373 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 996589dbb2419ea602bc5e348cbd05df 2024-11-26T21:49:53,374 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1732657792023.f0a1ab633a90e6b5f5e9ba8ce6f66010. 2024-11-26T21:49:53,374 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,fff,1732657792023.f0a1ab633a90e6b5f5e9ba8ce6f66010. 2024-11-26T21:49:53,374 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure cdd2d12bbd55678c7f638b4ef0fde3da, server=828b3f090c91,42273,1732657789877 because future has completed 2024-11-26T21:49:53,374 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,ooo,1732657792023.8a6bb8489f79432157c419605ac84965. 2024-11-26T21:49:53,374 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 8a6bb8489f79432157c419605ac84965, NAME => 'test,ooo,1732657792023.8a6bb8489f79432157c419605ac84965.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-26T21:49:53,375 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8a6bb8489f79432157c419605ac84965 2024-11-26T21:49:53,375 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,ooo,1732657792023.8a6bb8489f79432157c419605ac84965.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:53,375 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 8a6bb8489f79432157c419605ac84965 2024-11-26T21:49:53,375 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 8a6bb8489f79432157c419605ac84965 2024-11-26T21:49:53,375 INFO [StoreOpener-996589dbb2419ea602bc5e348cbd05df-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 996589dbb2419ea602bc5e348cbd05df 2024-11-26T21:49:53,377 INFO [StoreOpener-8a6bb8489f79432157c419605ac84965-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8a6bb8489f79432157c419605ac84965 2024-11-26T21:49:53,377 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=f0a1ab633a90e6b5f5e9ba8ce6f66010, regionState=OPEN, openSeqNum=2, regionLocation=828b3f090c91,40359,1732657789839 2024-11-26T21:49:53,377 INFO [StoreOpener-996589dbb2419ea602bc5e348cbd05df-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 996589dbb2419ea602bc5e348cbd05df columnFamilyName family 2024-11-26T21:49:53,377 DEBUG [StoreOpener-996589dbb2419ea602bc5e348cbd05df-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:53,378 INFO [StoreOpener-996589dbb2419ea602bc5e348cbd05df-1 {}] regionserver.HStore(327): Store=996589dbb2419ea602bc5e348cbd05df/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:49:53,378 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 996589dbb2419ea602bc5e348cbd05df 2024-11-26T21:49:53,380 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/996589dbb2419ea602bc5e348cbd05df 2024-11-26T21:49:53,380 INFO [StoreOpener-8a6bb8489f79432157c419605ac84965-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a6bb8489f79432157c419605ac84965 columnFamilyName family 2024-11-26T21:49:53,380 DEBUG [StoreOpener-8a6bb8489f79432157c419605ac84965-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:53,381 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure f0a1ab633a90e6b5f5e9ba8ce6f66010, server=828b3f090c91,40359,1732657789839 because future has completed 2024-11-26T21:49:53,381 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/996589dbb2419ea602bc5e348cbd05df 2024-11-26T21:49:53,381 INFO [StoreOpener-8a6bb8489f79432157c419605ac84965-1 {}] regionserver.HStore(327): Store=8a6bb8489f79432157c419605ac84965/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:49:53,381 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 8a6bb8489f79432157c419605ac84965 2024-11-26T21:49:53,381 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 996589dbb2419ea602bc5e348cbd05df 2024-11-26T21:49:53,381 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 996589dbb2419ea602bc5e348cbd05df 2024-11-26T21:49:53,382 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/8a6bb8489f79432157c419605ac84965 2024-11-26T21:49:53,383 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/8a6bb8489f79432157c419605ac84965 2024-11-26T21:49:53,384 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 8a6bb8489f79432157c419605ac84965 2024-11-26T21:49:53,384 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 8a6bb8489f79432157c419605ac84965 2024-11-26T21:49:53,384 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 996589dbb2419ea602bc5e348cbd05df 2024-11-26T21:49:53,386 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 8a6bb8489f79432157c419605ac84965 2024-11-26T21:49:53,386 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=14 2024-11-26T21:49:53,387 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure cdd2d12bbd55678c7f638b4ef0fde3da, server=828b3f090c91,42273,1732657789877 in 332 msec 2024-11-26T21:49:53,388 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/996589dbb2419ea602bc5e348cbd05df/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T21:49:53,389 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 996589dbb2419ea602bc5e348cbd05df; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65359978, jitterRate=-0.026060432195663452}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T21:49:53,390 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 996589dbb2419ea602bc5e348cbd05df 2024-11-26T21:49:53,390 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 996589dbb2419ea602bc5e348cbd05df: Running coprocessor pre-open hook at 1732657793373Writing region info on filesystem at 1732657793373Initializing all the Stores at 1732657793375 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657793375Cleaning up temporary data from old regions at 1732657793381 (+6 ms)Running coprocessor post-open hooks at 1732657793390 (+9 ms)Region opened successfully at 1732657793390 2024-11-26T21:49:53,390 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cdd2d12bbd55678c7f638b4ef0fde3da, ASSIGN in 624 msec 2024-11-26T21:49:53,391 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1732657792023.996589dbb2419ea602bc5e348cbd05df., pid=41, masterSystemTime=1732657793146 2024-11-26T21:49:53,391 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/8a6bb8489f79432157c419605ac84965/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T21:49:53,392 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 8a6bb8489f79432157c419605ac84965; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70537232, jitterRate=0.0510866641998291}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T21:49:53,392 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8a6bb8489f79432157c419605ac84965 2024-11-26T21:49:53,393 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 8a6bb8489f79432157c419605ac84965: Running coprocessor pre-open hook at 1732657793375Writing region info on filesystem at 1732657793375Initializing all the Stores at 1732657793376 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657793376Cleaning up temporary data from old regions at 1732657793384 (+8 ms)Running coprocessor post-open hooks at 1732657793392 (+8 ms)Region opened successfully at 1732657793392 2024-11-26T21:49:53,394 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1732657792023.8a6bb8489f79432157c419605ac84965., pid=55, masterSystemTime=1732657793190 2024-11-26T21:49:53,394 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1732657792023.996589dbb2419ea602bc5e348cbd05df. 2024-11-26T21:49:53,394 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,qqq,1732657792023.996589dbb2419ea602bc5e348cbd05df. 2024-11-26T21:49:53,395 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,uuu,1732657792023.317db83a078d27fdfa108a4454669013. 2024-11-26T21:49:53,395 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 317db83a078d27fdfa108a4454669013, NAME => 'test,uuu,1732657792023.317db83a078d27fdfa108a4454669013.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-26T21:49:53,395 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 317db83a078d27fdfa108a4454669013 2024-11-26T21:49:53,395 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,uuu,1732657792023.317db83a078d27fdfa108a4454669013.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:53,396 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 317db83a078d27fdfa108a4454669013 2024-11-26T21:49:53,396 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 317db83a078d27fdfa108a4454669013 2024-11-26T21:49:53,396 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=10 2024-11-26T21:49:53,396 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure f0a1ab633a90e6b5f5e9ba8ce6f66010, server=828b3f090c91,40359,1732657789839 in 329 msec 2024-11-26T21:49:53,397 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1732657792023.8a6bb8489f79432157c419605ac84965. 2024-11-26T21:49:53,397 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,ooo,1732657792023.8a6bb8489f79432157c419605ac84965. 2024-11-26T21:49:53,397 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,lll,1732657792023.ac402503c294574c0e555c45a12f0a4e. 2024-11-26T21:49:53,397 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => ac402503c294574c0e555c45a12f0a4e, NAME => 'test,lll,1732657792023.ac402503c294574c0e555c45a12f0a4e.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-26T21:49:53,397 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ac402503c294574c0e555c45a12f0a4e 2024-11-26T21:49:53,398 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,lll,1732657792023.ac402503c294574c0e555c45a12f0a4e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:53,398 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for ac402503c294574c0e555c45a12f0a4e 2024-11-26T21:49:53,398 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for ac402503c294574c0e555c45a12f0a4e 2024-11-26T21:49:53,398 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=8a6bb8489f79432157c419605ac84965, regionState=OPEN, openSeqNum=2, regionLocation=828b3f090c91,40359,1732657789839 2024-11-26T21:49:53,401 INFO [StoreOpener-ac402503c294574c0e555c45a12f0a4e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ac402503c294574c0e555c45a12f0a4e 2024-11-26T21:49:53,401 INFO [StoreOpener-317db83a078d27fdfa108a4454669013-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 317db83a078d27fdfa108a4454669013 2024-11-26T21:49:53,403 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a6bb8489f79432157c419605ac84965, server=828b3f090c91,40359,1732657789839 because future has completed 2024-11-26T21:49:53,404 INFO [StoreOpener-ac402503c294574c0e555c45a12f0a4e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ac402503c294574c0e555c45a12f0a4e columnFamilyName family 2024-11-26T21:49:53,404 INFO [StoreOpener-317db83a078d27fdfa108a4454669013-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 317db83a078d27fdfa108a4454669013 columnFamilyName family 2024-11-26T21:49:53,404 DEBUG [StoreOpener-ac402503c294574c0e555c45a12f0a4e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:53,404 DEBUG [StoreOpener-317db83a078d27fdfa108a4454669013-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:53,405 INFO [StoreOpener-ac402503c294574c0e555c45a12f0a4e-1 {}] regionserver.HStore(327): Store=ac402503c294574c0e555c45a12f0a4e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:49:53,405 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f0a1ab633a90e6b5f5e9ba8ce6f66010, ASSIGN in 639 msec 2024-11-26T21:49:53,405 INFO [StoreOpener-317db83a078d27fdfa108a4454669013-1 {}] regionserver.HStore(327): Store=317db83a078d27fdfa108a4454669013/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:49:53,405 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for ac402503c294574c0e555c45a12f0a4e 2024-11-26T21:49:53,406 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 317db83a078d27fdfa108a4454669013 2024-11-26T21:49:53,407 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/ac402503c294574c0e555c45a12f0a4e 2024-11-26T21:49:53,407 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/317db83a078d27fdfa108a4454669013 2024-11-26T21:49:53,407 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/317db83a078d27fdfa108a4454669013 2024-11-26T21:49:53,407 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/ac402503c294574c0e555c45a12f0a4e 2024-11-26T21:49:53,408 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 317db83a078d27fdfa108a4454669013 2024-11-26T21:49:53,408 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 317db83a078d27fdfa108a4454669013 2024-11-26T21:49:53,408 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for ac402503c294574c0e555c45a12f0a4e 2024-11-26T21:49:53,408 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for ac402503c294574c0e555c45a12f0a4e 2024-11-26T21:49:53,410 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for ac402503c294574c0e555c45a12f0a4e 2024-11-26T21:49:53,410 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 317db83a078d27fdfa108a4454669013 2024-11-26T21:49:53,413 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=19 2024-11-26T21:49:53,413 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 8a6bb8489f79432157c419605ac84965, server=828b3f090c91,40359,1732657789839 in 328 msec 2024-11-26T21:49:53,413 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/ac402503c294574c0e555c45a12f0a4e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T21:49:53,413 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/317db83a078d27fdfa108a4454669013/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T21:49:53,414 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 317db83a078d27fdfa108a4454669013; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69530604, jitterRate=0.03608673810958862}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T21:49:53,414 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened ac402503c294574c0e555c45a12f0a4e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70124301, jitterRate=0.044933512806892395}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T21:49:53,414 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 317db83a078d27fdfa108a4454669013 2024-11-26T21:49:53,414 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ac402503c294574c0e555c45a12f0a4e 2024-11-26T21:49:53,414 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 317db83a078d27fdfa108a4454669013: Running coprocessor pre-open hook at 1732657793396Writing region info on filesystem at 1732657793396Initializing all the Stores at 1732657793398 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657793398Cleaning up temporary data from old regions at 1732657793408 (+10 ms)Running coprocessor post-open hooks at 1732657793414 (+6 ms)Region opened successfully at 1732657793414 2024-11-26T21:49:53,414 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for ac402503c294574c0e555c45a12f0a4e: Running coprocessor pre-open hook at 1732657793398Writing region info on filesystem at 1732657793398Initializing all the Stores at 1732657793400 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657793400Cleaning up temporary data from old regions at 1732657793408 (+8 ms)Running coprocessor post-open hooks at 1732657793414 (+6 ms)Region opened successfully at 1732657793414 2024-11-26T21:49:53,415 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1732657792023.317db83a078d27fdfa108a4454669013., pid=42, masterSystemTime=1732657793146 2024-11-26T21:49:53,415 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1732657792023.ac402503c294574c0e555c45a12f0a4e., pid=54, masterSystemTime=1732657793190 2024-11-26T21:49:53,417 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8a6bb8489f79432157c419605ac84965, ASSIGN in 651 msec 2024-11-26T21:49:53,418 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1732657792023.317db83a078d27fdfa108a4454669013. 2024-11-26T21:49:53,418 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,uuu,1732657792023.317db83a078d27fdfa108a4454669013. 2024-11-26T21:49:53,419 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=996589dbb2419ea602bc5e348cbd05df, regionState=OPEN, openSeqNum=2, regionLocation=828b3f090c91,42273,1732657789877 2024-11-26T21:49:53,420 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1732657792023.ac402503c294574c0e555c45a12f0a4e. 2024-11-26T21:49:53,420 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,lll,1732657792023.ac402503c294574c0e555c45a12f0a4e. 2024-11-26T21:49:53,420 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,www,1732657792023.f45cc001ffd649bcfbfe0f7af1630961. 2024-11-26T21:49:53,420 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => f45cc001ffd649bcfbfe0f7af1630961, NAME => 'test,www,1732657792023.f45cc001ffd649bcfbfe0f7af1630961.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-26T21:49:53,421 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f45cc001ffd649bcfbfe0f7af1630961 2024-11-26T21:49:53,421 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,www,1732657792023.f45cc001ffd649bcfbfe0f7af1630961.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:53,421 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for f45cc001ffd649bcfbfe0f7af1630961 2024-11-26T21:49:53,421 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for f45cc001ffd649bcfbfe0f7af1630961 2024-11-26T21:49:53,422 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 996589dbb2419ea602bc5e348cbd05df, server=828b3f090c91,42273,1732657789877 because future has completed 2024-11-26T21:49:53,423 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=ac402503c294574c0e555c45a12f0a4e, regionState=OPEN, openSeqNum=2, regionLocation=828b3f090c91,40359,1732657789839 2024-11-26T21:49:53,424 INFO [StoreOpener-f45cc001ffd649bcfbfe0f7af1630961-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f45cc001ffd649bcfbfe0f7af1630961 2024-11-26T21:49:53,426 INFO [StoreOpener-f45cc001ffd649bcfbfe0f7af1630961-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f45cc001ffd649bcfbfe0f7af1630961 columnFamilyName family 2024-11-26T21:49:53,426 DEBUG [StoreOpener-f45cc001ffd649bcfbfe0f7af1630961-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:53,426 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure ac402503c294574c0e555c45a12f0a4e, server=828b3f090c91,40359,1732657789839 because future has completed 2024-11-26T21:49:53,427 INFO [StoreOpener-f45cc001ffd649bcfbfe0f7af1630961-1 {}] regionserver.HStore(327): Store=f45cc001ffd649bcfbfe0f7af1630961/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:49:53,427 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for f45cc001ffd649bcfbfe0f7af1630961 2024-11-26T21:49:53,428 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/f45cc001ffd649bcfbfe0f7af1630961 2024-11-26T21:49:53,429 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/f45cc001ffd649bcfbfe0f7af1630961 2024-11-26T21:49:53,430 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for f45cc001ffd649bcfbfe0f7af1630961 2024-11-26T21:49:53,430 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for f45cc001ffd649bcfbfe0f7af1630961 2024-11-26T21:49:53,431 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=21 2024-11-26T21:49:53,431 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 996589dbb2419ea602bc5e348cbd05df, server=828b3f090c91,42273,1732657789877 in 424 msec 2024-11-26T21:49:53,432 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for f45cc001ffd649bcfbfe0f7af1630961 2024-11-26T21:49:53,433 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=996589dbb2419ea602bc5e348cbd05df, ASSIGN in 668 msec 2024-11-26T21:49:53,435 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/f45cc001ffd649bcfbfe0f7af1630961/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T21:49:53,436 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened f45cc001ffd649bcfbfe0f7af1630961; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64865480, jitterRate=-0.03342902660369873}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T21:49:53,436 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f45cc001ffd649bcfbfe0f7af1630961 2024-11-26T21:49:53,436 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for f45cc001ffd649bcfbfe0f7af1630961: Running coprocessor pre-open hook at 1732657793421Writing region info on filesystem at 1732657793421Initializing all the Stores at 1732657793423 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657793423Cleaning up temporary data from old regions at 1732657793430 (+7 ms)Running coprocessor post-open hooks at 1732657793436 (+6 ms)Region opened successfully at 1732657793436 2024-11-26T21:49:53,437 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1732657792023.f45cc001ffd649bcfbfe0f7af1630961., pid=46, masterSystemTime=1732657793190 2024-11-26T21:49:53,439 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=16 2024-11-26T21:49:53,440 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure ac402503c294574c0e555c45a12f0a4e, server=828b3f090c91,40359,1732657789839 in 360 msec 2024-11-26T21:49:53,440 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1732657792023.f45cc001ffd649bcfbfe0f7af1630961. 2024-11-26T21:49:53,440 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,www,1732657792023.f45cc001ffd649bcfbfe0f7af1630961. 2024-11-26T21:49:53,440 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,iii,1732657792023.d22f8ae31d7e7a4d0c284e3b9676a393. 2024-11-26T21:49:53,441 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => d22f8ae31d7e7a4d0c284e3b9676a393, NAME => 'test,iii,1732657792023.d22f8ae31d7e7a4d0c284e3b9676a393.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-26T21:49:53,441 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d22f8ae31d7e7a4d0c284e3b9676a393 2024-11-26T21:49:53,441 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,iii,1732657792023.d22f8ae31d7e7a4d0c284e3b9676a393.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:53,441 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for d22f8ae31d7e7a4d0c284e3b9676a393 2024-11-26T21:49:53,441 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for d22f8ae31d7e7a4d0c284e3b9676a393 2024-11-26T21:49:53,443 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=f45cc001ffd649bcfbfe0f7af1630961, regionState=OPEN, openSeqNum=2, regionLocation=828b3f090c91,40359,1732657789839 2024-11-26T21:49:53,443 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44203 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=828b3f090c91,42273,1732657789877, table=test, region=d1c5d2bc20a7b33de1c8b5a53f7fee57. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T21:49:53,444 INFO [StoreOpener-d22f8ae31d7e7a4d0c284e3b9676a393-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d22f8ae31d7e7a4d0c284e3b9676a393 2024-11-26T21:49:53,446 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44203 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=828b3f090c91,40359,1732657789839, table=test, region=f45cc001ffd649bcfbfe0f7af1630961. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T21:49:53,446 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44203 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=828b3f090c91,40359,1732657789839, table=test, region=1062179709b27df36edcb7f6b054a4a6. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T21:49:53,448 INFO [StoreOpener-d22f8ae31d7e7a4d0c284e3b9676a393-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d22f8ae31d7e7a4d0c284e3b9676a393 columnFamilyName family 2024-11-26T21:49:53,448 DEBUG [StoreOpener-d22f8ae31d7e7a4d0c284e3b9676a393-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:53,448 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure f45cc001ffd649bcfbfe0f7af1630961, server=828b3f090c91,40359,1732657789839 because future has completed 2024-11-26T21:49:53,449 INFO [StoreOpener-d22f8ae31d7e7a4d0c284e3b9676a393-1 {}] regionserver.HStore(327): Store=d22f8ae31d7e7a4d0c284e3b9676a393/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:49:53,449 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for d22f8ae31d7e7a4d0c284e3b9676a393 2024-11-26T21:49:53,450 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/d22f8ae31d7e7a4d0c284e3b9676a393 2024-11-26T21:49:53,451 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/d22f8ae31d7e7a4d0c284e3b9676a393 2024-11-26T21:49:53,451 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for d22f8ae31d7e7a4d0c284e3b9676a393 2024-11-26T21:49:53,452 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for d22f8ae31d7e7a4d0c284e3b9676a393 2024-11-26T21:49:53,454 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for d22f8ae31d7e7a4d0c284e3b9676a393 2024-11-26T21:49:53,455 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44203 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=828b3f090c91,46545,1732657789755, table=test, region=334949367d92cc18a393737b69358f37. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T21:49:53,455 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44203 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=828b3f090c91,46545,1732657789755, table=test, region=21af909e3b8cf553daecee4e59c0898b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T21:49:53,456 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ac402503c294574c0e555c45a12f0a4e, ASSIGN in 686 msec 2024-11-26T21:49:53,458 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/d22f8ae31d7e7a4d0c284e3b9676a393/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T21:49:53,458 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened d22f8ae31d7e7a4d0c284e3b9676a393; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68105804, jitterRate=0.014855563640594482}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T21:49:53,459 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d22f8ae31d7e7a4d0c284e3b9676a393 2024-11-26T21:49:53,459 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for d22f8ae31d7e7a4d0c284e3b9676a393: Running coprocessor pre-open hook at 1732657793442Writing region info on filesystem at 1732657793442Initializing all the Stores at 1732657793444 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657793444Cleaning up temporary data from old regions at 1732657793452 (+8 ms)Running coprocessor post-open hooks at 1732657793459 (+7 ms)Region opened successfully at 1732657793459 2024-11-26T21:49:53,460 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1732657792023.d22f8ae31d7e7a4d0c284e3b9676a393., pid=53, masterSystemTime=1732657793190 2024-11-26T21:49:53,463 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1732657792023.d22f8ae31d7e7a4d0c284e3b9676a393. 2024-11-26T21:49:53,463 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,iii,1732657792023.d22f8ae31d7e7a4d0c284e3b9676a393. 2024-11-26T21:49:53,464 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=27 2024-11-26T21:49:53,464 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure f45cc001ffd649bcfbfe0f7af1630961, server=828b3f090c91,40359,1732657789839 in 424 msec 2024-11-26T21:49:53,467 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f45cc001ffd649bcfbfe0f7af1630961, ASSIGN in 701 msec 2024-11-26T21:49:53,468 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=d22f8ae31d7e7a4d0c284e3b9676a393, regionState=OPEN, openSeqNum=2, regionLocation=828b3f090c91,40359,1732657789839 2024-11-26T21:49:53,471 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure d22f8ae31d7e7a4d0c284e3b9676a393, server=828b3f090c91,40359,1732657789839 because future has completed 2024-11-26T21:49:53,472 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=317db83a078d27fdfa108a4454669013, regionState=OPEN, openSeqNum=2, regionLocation=828b3f090c91,42273,1732657789877 2024-11-26T21:49:53,475 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 317db83a078d27fdfa108a4454669013, server=828b3f090c91,42273,1732657789877 because future has completed 2024-11-26T21:49:53,479 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=13 2024-11-26T21:49:53,479 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure d22f8ae31d7e7a4d0c284e3b9676a393, server=828b3f090c91,40359,1732657789839 in 404 msec 2024-11-26T21:49:53,482 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d22f8ae31d7e7a4d0c284e3b9676a393, ASSIGN in 716 msec 2024-11-26T21:49:53,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=25 2024-11-26T21:49:53,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 317db83a078d27fdfa108a4454669013, server=828b3f090c91,42273,1732657789877 in 475 msec 2024-11-26T21:49:53,487 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=317db83a078d27fdfa108a4454669013, ASSIGN in 723 msec 2024-11-26T21:49:53,488 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=0e9187292024abc7067d470edad3ae67, regionState=OPEN, openSeqNum=2, regionLocation=828b3f090c91,42273,1732657789877 2024-11-26T21:49:53,491 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0e9187292024abc7067d470edad3ae67, server=828b3f090c91,42273,1732657789877 because future has completed 2024-11-26T21:49:53,495 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=20 2024-11-26T21:49:53,495 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure d1c5d2bc20a7b33de1c8b5a53f7fee57, server=828b3f090c91,42273,1732657789877 in 494 msec 2024-11-26T21:49:53,498 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d1c5d2bc20a7b33de1c8b5a53f7fee57, ASSIGN in 732 msec 2024-11-26T21:49:53,502 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=28 2024-11-26T21:49:53,502 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure 0e9187292024abc7067d470edad3ae67, server=828b3f090c91,42273,1732657789877 in 485 msec 2024-11-26T21:49:53,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=0e9187292024abc7067d470edad3ae67, ASSIGN in 739 msec 2024-11-26T21:49:53,506 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=7a7c374815f888db179beadaf2489ad7, regionState=OPEN, openSeqNum=2, regionLocation=828b3f090c91,40359,1732657789839 2024-11-26T21:49:53,509 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7a7c374815f888db179beadaf2489ad7, server=828b3f090c91,40359,1732657789839 because future has completed 2024-11-26T21:49:53,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=30 2024-11-26T21:49:53,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure 1062179709b27df36edcb7f6b054a4a6, server=828b3f090c91,40359,1732657789839 in 467 msec 2024-11-26T21:49:53,514 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=1062179709b27df36edcb7f6b054a4a6, ASSIGN in 748 msec 2024-11-26T21:49:53,518 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=8 2024-11-26T21:49:53,518 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 7a7c374815f888db179beadaf2489ad7, server=828b3f090c91,40359,1732657789839 in 447 msec 2024-11-26T21:49:53,520 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7a7c374815f888db179beadaf2489ad7, ASSIGN in 755 msec 2024-11-26T21:49:53,523 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=26 2024-11-26T21:49:53,524 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 21af909e3b8cf553daecee4e59c0898b, server=828b3f090c91,46545,1732657789755 in 544 msec 2024-11-26T21:49:53,525 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN in 761 msec 2024-11-26T21:49:53,526 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=1ef98738dc2598149d89e210ccdb4e6f, regionState=OPEN, openSeqNum=2, regionLocation=828b3f090c91,40359,1732657789839 2024-11-26T21:49:53,529 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ef98738dc2598149d89e210ccdb4e6f, server=828b3f090c91,40359,1732657789839 because future has completed 2024-11-26T21:49:53,531 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=15 2024-11-26T21:49:53,531 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure 334949367d92cc18a393737b69358f37, server=828b3f090c91,46545,1732657789755 in 579 msec 2024-11-26T21:49:53,533 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN in 768 msec 2024-11-26T21:49:53,537 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=24 2024-11-26T21:49:53,537 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 1ef98738dc2598149d89e210ccdb4e6f, server=828b3f090c91,40359,1732657789839 in 476 msec 2024-11-26T21:49:53,540 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=4 2024-11-26T21:49:53,540 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=1ef98738dc2598149d89e210ccdb4e6f, ASSIGN in 774 msec 2024-11-26T21:49:53,541 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T21:49:53,541 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732657793541"}]},"ts":"1732657793541"} 2024-11-26T21:49:53,544 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-11-26T21:49:53,546 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T21:49:53,553 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 1.5140 sec 2024-11-26T21:49:54,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T21:49:54,200 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-11-26T21:49:54,200 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-11-26T21:49:54,202 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T21:49:54,223 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-11-26T21:49:54,224 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T21:49:54,225 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-11-26T21:49:54,225 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T21:49:54,231 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-26T21:49:54,256 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-26T21:49:54,278 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T21:49:54,280 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57780, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T21:49:54,282 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T21:49:54,284 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58040, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T21:49:54,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 36699c847bc0e6c6b06fd8533d3c9edb changed from -1.0 to 0.0, refreshing cache 2024-11-26T21:49:54,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 7a7c374815f888db179beadaf2489ad7 changed from -1.0 to 0.0, refreshing cache 2024-11-26T21:49:54,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f0a1ab633a90e6b5f5e9ba8ce6f66010 changed from -1.0 to 0.0, refreshing cache 2024-11-26T21:49:54,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d22f8ae31d7e7a4d0c284e3b9676a393 changed from -1.0 to 0.0, refreshing cache 2024-11-26T21:49:54,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ac402503c294574c0e555c45a12f0a4e changed from -1.0 to 0.0, refreshing cache 2024-11-26T21:49:54,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8a6bb8489f79432157c419605ac84965 changed from -1.0 to 0.0, refreshing cache 2024-11-26T21:49:54,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1ef98738dc2598149d89e210ccdb4e6f changed from -1.0 to 0.0, refreshing cache 2024-11-26T21:49:54,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f45cc001ffd649bcfbfe0f7af1630961 changed from -1.0 to 0.0, refreshing cache 2024-11-26T21:49:54,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1062179709b27df36edcb7f6b054a4a6 changed from -1.0 to 0.0, refreshing cache 2024-11-26T21:49:54,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 34e003c01eb3f0a2e10e33152402164c changed from -1.0 to 0.0, refreshing cache 2024-11-26T21:49:54,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8be22ac0f4cc7a17874b9d10ba204518 changed from -1.0 to 0.0, refreshing cache 2024-11-26T21:49:54,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region cdd2d12bbd55678c7f638b4ef0fde3da changed from -1.0 to 0.0, refreshing cache 2024-11-26T21:49:54,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 06a1142ff2ac98bd39e6f73547d6aa98 changed from -1.0 to 0.0, refreshing cache 2024-11-26T21:49:54,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d1c5d2bc20a7b33de1c8b5a53f7fee57 changed from -1.0 to 0.0, refreshing cache 2024-11-26T21:49:54,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 996589dbb2419ea602bc5e348cbd05df changed from -1.0 to 0.0, refreshing cache 2024-11-26T21:49:54,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 30cf2d9c6c6cf36bd61bb35f51df8f42 changed from -1.0 to 0.0, refreshing cache 2024-11-26T21:49:54,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 317db83a078d27fdfa108a4454669013 changed from -1.0 to 0.0, refreshing cache 2024-11-26T21:49:54,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0e9187292024abc7067d470edad3ae67 changed from -1.0 to 0.0, refreshing cache 2024-11-26T21:49:54,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region bf200defbfcf83538e813f9fa91d9c35 changed from -1.0 to 0.0, refreshing cache 2024-11-26T21:49:54,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 651cf53d9624e6cc98e46d6578c71496 changed from -1.0 to 0.0, refreshing cache 2024-11-26T21:49:54,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a2d6b8d1f56582e3f52e3cfe7a116c1d changed from -1.0 to 0.0, refreshing cache 2024-11-26T21:49:54,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 334949367d92cc18a393737b69358f37 changed from -1.0 to 0.0, refreshing cache 2024-11-26T21:49:54,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c4145a9c4106377623484a3eb786fe8a changed from -1.0 to 0.0, refreshing cache 2024-11-26T21:49:54,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2b8b8290cc9d27b8d226e5dc2b7d770d changed from -1.0 to 0.0, refreshing cache 2024-11-26T21:49:54,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 21af909e3b8cf553daecee4e59c0898b changed from -1.0 to 0.0, refreshing cache 2024-11-26T21:49:54,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1e7efa0e8c09fd1f0e1a765d7fc38181 changed from -1.0 to 0.0, refreshing cache 2024-11-26T21:49:54,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-11-26T21:49:54,355 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-11-26T21:49:54,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-11-26T21:49:54,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] balancer.BalancerClusterState(204): Hosts are {828b3f090c91=0} racks are {/default-rack=0} 2024-11-26T21:49:54,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] balancer.BalancerClusterState(303): server 0 has 9 regions 2024-11-26T21:49:54,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] balancer.BalancerClusterState(303): server 1 has 9 regions 2024-11-26T21:49:54,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] balancer.BalancerClusterState(303): server 2 has 9 regions 2024-11-26T21:49:54,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T21:49:54,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T21:49:54,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-26T21:49:54,357 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T21:49:54,357 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T21:49:54,357 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-26T21:49:54,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-11-26T21:49:54,362 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] balancer.StochasticLoadBalancer(395): Cluster wide - skipping load balancing because weighted average imbalance=0.016474464579901153 <= threshold(0.025). If you want more aggressive balancing, either lower hbase.master.balancer.stochastic.minCostNeedBalance from 0.025 or increase the relative multiplier(s) of the specific cost function(s). functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-11-26T21:49:54,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-11-26T21:49:54,365 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T21:49:54,375 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.2 set balanceSwitch=false 2024-11-26T21:49:54,394 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=386, OpenFileDescriptor=649, MaxFileDescriptor=1048576, SystemLoadAverage=234, ProcessCount=11, AvailableMemoryMB=11124 2024-11-26T21:49:54,399 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-11-26T21:49:54,400 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-11-26T21:49:54,400 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 828b3f090c91,46545,1732657789755 2024-11-26T21:49:54,401 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 828b3f090c91,46545,1732657789755: Simulated kill ***** 2024-11-26T21:49:54,401 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-26T21:49:54,404 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-26T21:49:54,407 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-26T21:49:54,407 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-26T21:49:54,408 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-26T21:49:54,408 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "Verbose": false, "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 306718976 }, "NonHeapMemoryUsage": { "committed": 151388160, "init": 7667712, "max": -1, "used": 149039632 }, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "828b3f090c91" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "828b3f090c91" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "828b3f090c91" } ] } 2024-11-26T21:49:54,412 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44203 {}] master.MasterRpcServices(700): 828b3f090c91,46545,1732657789755 reported a fatal error: ***** ABORTING region server 828b3f090c91,46545,1732657789755: Simulated kill ***** 2024-11-26T21:49:54,414 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '828b3f090c91,46545,1732657789755' ***** 2024-11-26T21:49:54,414 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-26T21:49:54,415 INFO [RS:0;828b3f090c91:46545 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T21:49:54,415 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T21:49:54,415 INFO [RS:0;828b3f090c91:46545 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-26T21:49:54,415 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T21:49:54,416 INFO [RS:0;828b3f090c91:46545 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-26T21:49:54,416 INFO [RS:0;828b3f090c91:46545 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T21:49:54,416 INFO [RS:0;828b3f090c91:46545 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;828b3f090c91:46545. 2024-11-26T21:49:54,416 DEBUG [RS:0;828b3f090c91:46545 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T21:49:54,423 DEBUG [RS:0;828b3f090c91:46545 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T21:49:54,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741836_1012 (size=32114) 2024-11-26T21:49:54,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741836_1012 (size=32114) 2024-11-26T21:49:54,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741836_1012 (size=32114) 2024-11-26T21:49:54,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741833_1009 (size=2232) 2024-11-26T21:49:54,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741833_1009 (size=2232) 2024-11-26T21:49:54,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741833_1009 (size=2232) 2024-11-26T21:49:54,437 DEBUG [RS:0;828b3f090c91:46545 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T21:49:54,437 INFO [RS:0;828b3f090c91:46545 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T21:49:54,437 INFO [RS:0;828b3f090c91:46545 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46545 2024-11-26T21:49:54,441 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46545-0x1007e03121b0001, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/828b3f090c91,46545,1732657789755 2024-11-26T21:49:54,441 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T21:49:54,441 INFO [RS:0;828b3f090c91:46545 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T21:49:54,443 INFO [regionserver/828b3f090c91:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T21:49:54,445 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [828b3f090c91,46545,1732657789755] 2024-11-26T21:49:54,446 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/828b3f090c91,46545,1732657789755 already deleted, retry=false 2024-11-26T21:49:54,446 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 828b3f090c91,46545,1732657789755 on 828b3f090c91,44203,1732657789166 2024-11-26T21:49:54,451 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true 2024-11-26T21:49:54,456 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=57 for 828b3f090c91,46545,1732657789755 (carryingMeta=true) 828b3f090c91,46545,1732657789755/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@43bf043a[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-26T21:49:54,457 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true 2024-11-26T21:49:54,459 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true, isMeta: true 2024-11-26T21:49:54,461 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,46545,1732657789755-splitting 2024-11-26T21:49:54,463 INFO [PEWorker-1 {}] master.SplitWALManager(105): 828b3f090c91,46545,1732657789755 WAL count=1, meta=true 2024-11-26T21:49:54,466 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta}] 2024-11-26T21:49:54,472 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:54,475 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877}] 2024-11-26T21:49:54,517 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T21:49:54,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46545-0x1007e03121b0001, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T21:49:54,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46545-0x1007e03121b0001, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T21:49:54,545 INFO [RS:0;828b3f090c91:46545 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T21:49:54,545 INFO [RS:0;828b3f090c91:46545 {}] regionserver.HRegionServer(1031): Exiting; stopping=828b3f090c91,46545,1732657789755; zookeeper connection closed. 2024-11-26T21:49:54,545 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3f7c68f8 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3f7c68f8 2024-11-26T21:49:54,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=59 2024-11-26T21:49:54,667 INFO [RS_LOG_REPLAY_OPS-regionserver/828b3f090c91:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(299): Splitting hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,46545,1732657789755-splitting/828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, size=31.4 K (32114bytes) 2024-11-26T21:49:54,668 DEBUG [RS_LOG_REPLAY_OPS-regionserver/828b3f090c91:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-26T21:49:54,669 DEBUG [RS_LOG_REPLAY_OPS-regionserver/828b3f090c91:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-26T21:49:54,669 INFO [RS_LOG_REPLAY_OPS-regionserver/828b3f090c91:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,46545,1732657789755-splitting/828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta 2024-11-26T21:49:54,675 INFO [RS_LOG_REPLAY_OPS-regionserver/828b3f090c91:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,46545,1732657789755-splitting/828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta after 4ms 2024-11-26T21:49:54,684 INFO [RS_LOG_REPLAY_OPS-regionserver/828b3f090c91:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(310): Open hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,46545,1732657789755-splitting/828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta took 18ms 2024-11-26T21:49:54,694 DEBUG [RS_LOG_REPLAY_OPS-regionserver/828b3f090c91:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-26T21:49:54,700 INFO [RS_LOG_REPLAY_OPS-regionserver/828b3f090c91:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta.temp 2024-11-26T21:49:54,707 DEBUG [RS_LOG_REPLAY_OPS-regionserver/828b3f090c91:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,46545,1732657789755-splitting/828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta so closing down 2024-11-26T21:49:54,707 DEBUG [RS_LOG_REPLAY_OPS-regionserver/828b3f090c91:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T21:49:54,711 INFO [RS_LOG_REPLAY_OPS-regionserver/828b3f090c91:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta.temp 2024-11-26T21:49:54,715 INFO [RS_LOG_REPLAY_OPS-regionserver/828b3f090c91:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T21:49:54,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741864_1040 (size=31873) 2024-11-26T21:49:54,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741864_1040 (size=31873) 2024-11-26T21:49:54,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741864_1040 (size=31873) 2024-11-26T21:49:54,725 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta.temp (wrote 57 edits, skipped 0 edits in 2 ms) 2024-11-26T21:49:54,727 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta.temp to hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-26T21:49:54,731 INFO [RS_LOG_REPLAY_OPS-regionserver/828b3f090c91:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(425): Processed 58 edits across 1 Regions in 41 ms; skipped=1; WAL=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,46545,1732657789755-splitting/828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false 2024-11-26T21:49:54,732 DEBUG [RS_LOG_REPLAY_OPS-regionserver/828b3f090c91:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(428): Completed split of hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,46545,1732657789755-splitting/828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, journal: Splitting hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,46545,1732657789755-splitting/828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, size=31.4 K (32114bytes) at 1732657794668Finishing writing output for hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,46545,1732657789755-splitting/828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta so closing down at 1732657794707 (+39 ms)Creating recovered edits writer path=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta.temp at 1732657794712 (+5 ms)3 split writer threads finished at 1732657794715 (+3 ms)Closed recovered edits writer path=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta.temp (wrote 57 edits, skipped 0 edits in 2 ms) at 1732657794725 (+10 ms)Rename recovered edits hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta.temp to hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 at 1732657794727 (+2 ms)Processed 58 edits across 1 Regions in 41 ms; skipped=1; WAL=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,46545,1732657789755-splitting/828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false at 1732657794732 (+5 ms) 2024-11-26T21:49:54,732 DEBUG [RS_LOG_REPLAY_OPS-regionserver/828b3f090c91:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,46545,1732657789755-splitting/828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta 2024-11-26T21:49:54,734 DEBUG [RS_LOG_REPLAY_OPS-regionserver/828b3f090c91:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-26T21:49:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-26T21:49:54,744 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,46545,1732657789755-splitting/828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta to hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/oldWALs 2024-11-26T21:49:54,747 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-26T21:49:54,747 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 in 269 msec 2024-11-26T21:49:54,749 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:54,753 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-26T21:49:54,753 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 in 285 msec 2024-11-26T21:49:54,755 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,46545,1732657789755-splitting dir is empty, no logs to split. 2024-11-26T21:49:54,756 INFO [PEWorker-1 {}] master.SplitWALManager(105): 828b3f090c91,46545,1732657789755 WAL count=0, meta=true 2024-11-26T21:49:54,756 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 828b3f090c91,46545,1732657789755 WAL splitting is done? wals=0, meta=true 2024-11-26T21:49:54,758 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-26T21:49:54,760 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T21:49:54,761 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-26T21:49:54,911 DEBUG [828b3f090c91:44203 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-26T21:49:54,912 DEBUG [828b3f090c91:44203 {}] balancer.BalancerClusterState(204): Hosts are {828b3f090c91=0} racks are {/default-rack=0} 2024-11-26T21:49:54,912 DEBUG [828b3f090c91:44203 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T21:49:54,912 DEBUG [828b3f090c91:44203 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T21:49:54,912 DEBUG [828b3f090c91:44203 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T21:49:54,912 DEBUG [828b3f090c91:44203 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T21:49:54,913 INFO [828b3f090c91:44203 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T21:49:54,913 INFO [828b3f090c91:44203 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T21:49:54,913 DEBUG [828b3f090c91:44203 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T21:49:54,914 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=828b3f090c91,40359,1732657789839 2024-11-26T21:49:54,917 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 828b3f090c91,40359,1732657789839, state=OPENING 2024-11-26T21:49:54,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:54,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:54,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:54,919 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:54,919 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:54,919 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:54,919 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T21:49:54,920 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,40359,1732657789839}] 2024-11-26T21:49:54,990 INFO [regionserver/828b3f090c91:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-26T21:49:54,990 INFO [regionserver/828b3f090c91:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-26T21:49:55,078 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-26T21:49:55,078 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T21:49:55,078 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-26T21:49:55,081 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=828b3f090c91%2C40359%2C1732657789839.meta, suffix=.meta, logDir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,40359,1732657789839, archiveDir=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/oldWALs, maxLogs=32 2024-11-26T21:49:55,100 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,40359,1732657789839/828b3f090c91%2C40359%2C1732657789839.meta.1732657795083.meta, exclude list is [], retry=0 2024-11-26T21:49:55,104 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43659,DS-7dc835d1-8731-44e2-b974-f85d5c6f47f1,DISK] 2024-11-26T21:49:55,104 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40345,DS-4fbda298-e9f3-436b-9808-ab5ec6eece62,DISK] 2024-11-26T21:49:55,104 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39537,DS-aed5038f-ce60-4629-a808-1e2f6bde72d8,DISK] 2024-11-26T21:49:55,108 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,40359,1732657789839/828b3f090c91%2C40359%2C1732657789839.meta.1732657795083.meta 2024-11-26T21:49:55,108 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36781:36781),(127.0.0.1/127.0.0.1:36805:36805),(127.0.0.1/127.0.0.1:45705:45705)] 2024-11-26T21:49:55,108 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-26T21:49:55,109 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-26T21:49:55,109 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-26T21:49:55,109 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-26T21:49:55,109 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-26T21:49:55,109 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:55,109 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-26T21:49:55,109 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-26T21:49:55,113 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T21:49:55,114 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T21:49:55,114 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:55,115 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T21:49:55,115 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T21:49:55,116 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T21:49:55,116 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:55,117 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T21:49:55,117 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T21:49:55,118 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T21:49:55,118 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:55,119 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T21:49:55,119 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T21:49:55,120 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T21:49:55,121 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:55,121 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T21:49:55,121 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T21:49:55,123 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/hbase/meta/1588230740 2024-11-26T21:49:55,126 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/hbase/meta/1588230740 2024-11-26T21:49:55,127 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-26T21:49:55,146 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5793): Applied 264, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=60, path=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-26T21:49:55,151 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=28 KB heapSize=48.31 KB 2024-11-26T21:49:55,241 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/hbase/meta/1588230740/.tmp/info/c07d5d1e82b1442dab6fb65cbf5850ee is 138, key is test,bbb,1732657792023.36699c847bc0e6c6b06fd8533d3c9edb./info:regioninfo/1732657793351/Put/seqid=0 2024-11-26T21:49:55,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741866_1042 (size=34924) 2024-11-26T21:49:55,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741866_1042 (size=34924) 2024-11-26T21:49:55,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741866_1042 (size=34924) 2024-11-26T21:49:55,668 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.86 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/hbase/meta/1588230740/.tmp/info/c07d5d1e82b1442dab6fb65cbf5850ee 2024-11-26T21:49:55,733 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/hbase/meta/1588230740/.tmp/ns/5124a62ac446406cb3f52729181ffe5f is 43, key is default/ns:d/1732657791751/Put/seqid=0 2024-11-26T21:49:55,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741867_1043 (size=5153) 2024-11-26T21:49:55,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741867_1043 (size=5153) 2024-11-26T21:49:55,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741867_1043 (size=5153) 2024-11-26T21:49:55,745 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/hbase/meta/1588230740/.tmp/ns/5124a62ac446406cb3f52729181ffe5f 2024-11-26T21:49:55,771 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/hbase/meta/1588230740/.tmp/table/4544e484baf74ea0ba82b2d8aa3ee253 is 40, key is test/table:state/1732657793541/Put/seqid=0 2024-11-26T21:49:55,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741868_1044 (size=5165) 2024-11-26T21:49:55,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741868_1044 (size=5165) 2024-11-26T21:49:55,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741868_1044 (size=5165) 2024-11-26T21:49:55,779 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/hbase/meta/1588230740/.tmp/table/4544e484baf74ea0ba82b2d8aa3ee253 2024-11-26T21:49:55,787 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/hbase/meta/1588230740/.tmp/info/c07d5d1e82b1442dab6fb65cbf5850ee as hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/hbase/meta/1588230740/info/c07d5d1e82b1442dab6fb65cbf5850ee 2024-11-26T21:49:55,797 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/hbase/meta/1588230740/info/c07d5d1e82b1442dab6fb65cbf5850ee, entries=260, sequenceid=60, filesize=34.1 K 2024-11-26T21:49:55,800 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/hbase/meta/1588230740/.tmp/ns/5124a62ac446406cb3f52729181ffe5f as hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/hbase/meta/1588230740/ns/5124a62ac446406cb3f52729181ffe5f 2024-11-26T21:49:55,809 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/hbase/meta/1588230740/ns/5124a62ac446406cb3f52729181ffe5f, entries=2, sequenceid=60, filesize=5.0 K 2024-11-26T21:49:55,810 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/hbase/meta/1588230740/.tmp/table/4544e484baf74ea0ba82b2d8aa3ee253 as hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/hbase/meta/1588230740/table/4544e484baf74ea0ba82b2d8aa3ee253 2024-11-26T21:49:55,819 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/hbase/meta/1588230740/table/4544e484baf74ea0ba82b2d8aa3ee253, entries=2, sequenceid=60, filesize=5.0 K 2024-11-26T21:49:55,823 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 672ms, sequenceid=60, compaction requested=false; wal=null 2024-11-26T21:49:55,824 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-26T21:49:55,827 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-26T21:49:55,829 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T21:49:55,829 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T21:49:55,830 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T21:49:55,832 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T21:49:55,835 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/hbase/meta/1588230740/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=1 2024-11-26T21:49:55,836 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=61; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70650929, jitterRate=0.052780881524086}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T21:49:55,836 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-26T21:49:55,837 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732657795109Writing region info on filesystem at 1732657795109Initializing all the Stores at 1732657795111 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732657795111Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732657795113 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657795113Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732657795113Obtaining lock to block concurrent updates at 1732657795151 (+38 ms)Preparing flush snapshotting stores in 1588230740 at 1732657795151Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=28672, getHeapSize=49408, getOffHeapSize=0, getCellsCount=264 at 1732657795156 (+5 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732657795156Flushing 1588230740/info: creating writer at 1732657795158 (+2 ms)Flushing 1588230740/info: appending metadata at 1732657795225 (+67 ms)Flushing 1588230740/info: closing flushed file at 1732657795229 (+4 ms)Flushing 1588230740/ns: creating writer at 1732657795715 (+486 ms)Flushing 1588230740/ns: appending metadata at 1732657795732 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1732657795732Flushing 1588230740/table: creating writer at 1732657795756 (+24 ms)Flushing 1588230740/table: appending metadata at 1732657795769 (+13 ms)Flushing 1588230740/table: closing flushed file at 1732657795769Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25db060e: reopening flushed file at 1732657795786 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ae9531: reopening flushed file at 1732657795798 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3eda2494: reopening flushed file at 1732657795809 (+11 ms)Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 672ms, sequenceid=60, compaction requested=false; wal=null at 1732657795823 (+14 ms)Cleaning up temporary data from old regions at 1732657795829 (+6 ms)Running coprocessor post-open hooks at 1732657795836 (+7 ms)Region opened successfully at 1732657795837 (+1 ms) 2024-11-26T21:49:55,838 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=61, masterSystemTime=1732657795072 2024-11-26T21:49:55,841 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-26T21:49:55,841 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-26T21:49:55,842 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=61, regionLocation=828b3f090c91,40359,1732657789839 2024-11-26T21:49:55,843 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 828b3f090c91,40359,1732657789839, state=OPEN 2024-11-26T21:49:55,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:55,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:55,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:55,844 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:55,844 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:55,844 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=61, ppid=60, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=828b3f090c91,40359,1732657789839 2024-11-26T21:49:55,844 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:55,847 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-26T21:49:55,847 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,40359,1732657789839 in 925 msec 2024-11-26T21:49:55,860 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=57 2024-11-26T21:49:55,860 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.0890 sec 2024-11-26T21:49:55,860 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 828b3f090c91,46545,1732657789755 had 9 regions 2024-11-26T21:49:55,862 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true, isMeta: false 2024-11-26T21:49:55,864 INFO [PEWorker-1 {}] master.SplitWALManager(105): 828b3f090c91,46545,1732657789755 WAL count=1, meta=false 2024-11-26T21:49:55,864 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102}] 2024-11-26T21:49:55,866 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=828b3f090c91,40359,1732657789839 2024-11-26T21:49:55,868 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839}] 2024-11-26T21:49:56,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40359 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=63 2024-11-26T21:49:56,045 INFO [RS_LOG_REPLAY_OPS-regionserver/828b3f090c91:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(299): Splitting hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,46545,1732657789755-splitting/828b3f090c91%2C46545%2C1732657789755.1732657791102, size=2.2 K (2232bytes) 2024-11-26T21:49:56,045 INFO [RS_LOG_REPLAY_OPS-regionserver/828b3f090c91:0-0 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,46545,1732657789755-splitting/828b3f090c91%2C46545%2C1732657789755.1732657791102 2024-11-26T21:49:56,046 INFO [RS_LOG_REPLAY_OPS-regionserver/828b3f090c91:0-0 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,46545,1732657789755-splitting/828b3f090c91%2C46545%2C1732657789755.1732657791102 after 1ms 2024-11-26T21:49:56,050 INFO [RS_LOG_REPLAY_OPS-regionserver/828b3f090c91:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(310): Open hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,46545,1732657789755-splitting/828b3f090c91%2C46545%2C1732657789755.1732657791102 took 5ms 2024-11-26T21:49:56,055 DEBUG [RS_LOG_REPLAY_OPS-regionserver/828b3f090c91:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for a2d6b8d1f56582e3f52e3cfe7a116c1d: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-26T21:49:56,057 DEBUG [RS_LOG_REPLAY_OPS-regionserver/828b3f090c91:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for bf200defbfcf83538e813f9fa91d9c35: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-26T21:49:56,060 DEBUG [RS_LOG_REPLAY_OPS-regionserver/828b3f090c91:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for c4145a9c4106377623484a3eb786fe8a: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-26T21:49:56,064 DEBUG [RS_LOG_REPLAY_OPS-regionserver/828b3f090c91:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 651cf53d9624e6cc98e46d6578c71496: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-26T21:49:56,067 DEBUG [RS_LOG_REPLAY_OPS-regionserver/828b3f090c91:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 334949367d92cc18a393737b69358f37: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-26T21:49:56,069 DEBUG [RS_LOG_REPLAY_OPS-regionserver/828b3f090c91:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 21af909e3b8cf553daecee4e59c0898b: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-26T21:49:56,071 DEBUG [RS_LOG_REPLAY_OPS-regionserver/828b3f090c91:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 2b8b8290cc9d27b8d226e5dc2b7d770d: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-26T21:49:56,073 DEBUG [RS_LOG_REPLAY_OPS-regionserver/828b3f090c91:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 1e7efa0e8c09fd1f0e1a765d7fc38181: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-26T21:49:56,074 DEBUG [RS_LOG_REPLAY_OPS-regionserver/828b3f090c91:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,46545,1732657789755-splitting/828b3f090c91%2C46545%2C1732657789755.1732657791102 so closing down 2024-11-26T21:49:56,074 DEBUG [RS_LOG_REPLAY_OPS-regionserver/828b3f090c91:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T21:49:56,074 INFO [RS_LOG_REPLAY_OPS-regionserver/828b3f090c91:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T21:49:56,074 INFO [RS_LOG_REPLAY_OPS-regionserver/828b3f090c91:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(425): Processed 8 edits across 0 Regions in 23 ms; skipped=8; WAL=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,46545,1732657789755-splitting/828b3f090c91%2C46545%2C1732657789755.1732657791102, size=2.2 K, length=2232, corrupted=false, cancelled=false 2024-11-26T21:49:56,074 DEBUG [RS_LOG_REPLAY_OPS-regionserver/828b3f090c91:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(428): Completed split of hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,46545,1732657789755-splitting/828b3f090c91%2C46545%2C1732657789755.1732657791102, journal: Splitting hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,46545,1732657789755-splitting/828b3f090c91%2C46545%2C1732657789755.1732657791102, size=2.2 K (2232bytes) at 1732657796045Finishing writing output for hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,46545,1732657789755-splitting/828b3f090c91%2C46545%2C1732657789755.1732657791102 so closing down at 1732657796074 (+29 ms)3 split writer threads finished at 1732657796074Processed 8 edits across 0 Regions in 23 ms; skipped=8; WAL=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,46545,1732657789755-splitting/828b3f090c91%2C46545%2C1732657789755.1732657791102, size=2.2 K, length=2232, corrupted=false, cancelled=false at 1732657796074 2024-11-26T21:49:56,074 DEBUG [RS_LOG_REPLAY_OPS-regionserver/828b3f090c91:0-0 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,46545,1732657789755-splitting/828b3f090c91%2C46545%2C1732657789755.1732657791102 2024-11-26T21:49:56,074 DEBUG [RS_LOG_REPLAY_OPS-regionserver/828b3f090c91:0-0 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-26T21:49:56,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44203 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-26T21:49:56,079 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,46545,1732657789755-splitting/828b3f090c91%2C46545%2C1732657789755.1732657791102 to hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/oldWALs 2024-11-26T21:49:56,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-26T21:49:56,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 in 212 msec 2024-11-26T21:49:56,084 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,087 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=57 2024-11-26T21:49:56,087 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 in 220 msec 2024-11-26T21:49:56,089 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/WALs/828b3f090c91,46545,1732657789755-splitting dir is empty, no logs to split. 2024-11-26T21:49:56,089 INFO [PEWorker-1 {}] master.SplitWALManager(105): 828b3f090c91,46545,1732657789755 WAL count=0, meta=false 2024-11-26T21:49:56,089 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 828b3f090c91,46545,1732657789755 WAL splitting is done? wals=0, meta=false 2024-11-26T21:49:56,092 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN}] 2024-11-26T21:49:56,095 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN 2024-11-26T21:49:56,097 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-26T21:49:56,099 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN 2024-11-26T21:49:56,101 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-26T21:49:56,103 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN 2024-11-26T21:49:56,104 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-26T21:49:56,105 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN 2024-11-26T21:49:56,106 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-26T21:49:56,107 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN 2024-11-26T21:49:56,108 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-26T21:49:56,109 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN 2024-11-26T21:49:56,110 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-26T21:49:56,112 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN 2024-11-26T21:49:56,113 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-26T21:49:56,114 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN 2024-11-26T21:49:56,115 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-26T21:49:56,248 DEBUG [828b3f090c91:44203 {}] balancer.BalancerClusterState(204): Hosts are {828b3f090c91=0} racks are {/default-rack=0} 2024-11-26T21:49:56,249 DEBUG [828b3f090c91:44203 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T21:49:56,249 DEBUG [828b3f090c91:44203 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T21:49:56,249 DEBUG [828b3f090c91:44203 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T21:49:56,249 DEBUG [828b3f090c91:44203 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T21:49:56,250 INFO [828b3f090c91:44203 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T21:49:56,250 INFO [828b3f090c91:44203 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T21:49:56,250 DEBUG [828b3f090c91:44203 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T21:49:56,251 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=21af909e3b8cf553daecee4e59c0898b, regionState=OPENING, regionLocation=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,253 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=c4145a9c4106377623484a3eb786fe8a, regionState=OPENING, regionLocation=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,254 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=a2d6b8d1f56582e3f52e3cfe7a116c1d, regionState=OPENING, regionLocation=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,254 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 828b3f090c91:46545 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 828b3f090c91/172.17.0.2:46545 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T21:49:56,256 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=bf200defbfcf83538e813f9fa91d9c35, regionState=OPENING, regionLocation=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,257 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=1e7efa0e8c09fd1f0e1a765d7fc38181, regionState=OPENING, regionLocation=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,258 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=828b3f090c91,46545,1732657789755, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=828b3f090c91,46545,1732657789755, seqNum=-1, error=java.net.ConnectException: Call to address=828b3f090c91:46545 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 828b3f090c91/172.17.0.2:46545 2024-11-26T21:49:56,258 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=2b8b8290cc9d27b8d226e5dc2b7d770d, regionState=OPENING, regionLocation=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,258 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=334949367d92cc18a393737b69358f37, regionState=OPENING, regionLocation=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,258 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=828b3f090c91,46545,1732657789755, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-26T21:49:56,259 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=828b3f090c91,46545,1732657789755, seqNum=-1 from cache 2024-11-26T21:49:56,259 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=651cf53d9624e6cc98e46d6578c71496, regionState=OPENING, regionLocation=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,259 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T21:49:56,259 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=828b3f090c91,40359,1732657789839, seqNum=-1] 2024-11-26T21:49:56,260 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 828b3f090c91:46545 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 828b3f090c91/172.17.0.2:46545 2024-11-26T21:49:56,260 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T21:49:56,260 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 828b3f090c91:46545 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 828b3f090c91/172.17.0.2:46545 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T21:49:56,261 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=828b3f090c91,46545,1732657789755, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=828b3f090c91,40359,1732657789839, seqNum=-1, error=java.net.ConnectException: Call to address=828b3f090c91:46545 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 828b3f090c91/172.17.0.2:46545 2024-11-26T21:49:56,261 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=828b3f090c91,46545,1732657789755, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=828b3f090c91,40359,1732657789839, seqNum=-1, error=java.net.ConnectException: Call to address=828b3f090c91:46545 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 828b3f090c91/172.17.0.2:46545 2024-11-26T21:49:56,261 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=828b3f090c91,46545,1732657789755, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=828b3f090c91,40359,1732657789839, seqNum=-1, error=java.net.ConnectException: Call to address=828b3f090c91:46545 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 828b3f090c91/172.17.0.2:46545 2024-11-26T21:49:56,261 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=828b3f090c91,46545,1732657789755, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=828b3f090c91,40359,1732657789839, seqNum=-1, error=java.net.ConnectException: Call to address=828b3f090c91:46545 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 828b3f090c91/172.17.0.2:46545 2024-11-26T21:49:56,261 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56205, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T21:49:56,261 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=828b3f090c91,46545,1732657789755, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=828b3f090c91,40359,1732657789839, seqNum=-1, error=java.net.ConnectException: Call to address=828b3f090c91:46545 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 828b3f090c91/172.17.0.2:46545 2024-11-26T21:49:56,261 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=828b3f090c91,46545,1732657789755, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=828b3f090c91,40359,1732657789839, seqNum=-1, error=java.net.ConnectException: Call to address=828b3f090c91:46545 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 828b3f090c91/172.17.0.2:46545 2024-11-26T21:49:56,262 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 828b3f090c91:46545 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 828b3f090c91/172.17.0.2:46545 2024-11-26T21:49:56,263 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN because future has completed 2024-11-26T21:49:56,264 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877}] 2024-11-26T21:49:56,372 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN because future has completed 2024-11-26T21:49:56,372 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21af909e3b8cf553daecee4e59c0898b, server=828b3f090c91,40359,1732657789839}] 2024-11-26T21:49:56,374 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN because future has completed 2024-11-26T21:49:56,375 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN because future has completed 2024-11-26T21:49:56,376 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN because future has completed 2024-11-26T21:49:56,378 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN because future has completed 2024-11-26T21:49:56,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e7efa0e8c09fd1f0e1a765d7fc38181, server=828b3f090c91,42273,1732657789877}] 2024-11-26T21:49:56,379 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN because future has completed 2024-11-26T21:49:56,380 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN because future has completed 2024-11-26T21:49:56,383 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 334949367d92cc18a393737b69358f37, server=828b3f090c91,42273,1732657789877}] 2024-11-26T21:49:56,386 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b8b8290cc9d27b8d226e5dc2b7d770d, server=828b3f090c91,42273,1732657789877}] 2024-11-26T21:49:56,390 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure bf200defbfcf83538e813f9fa91d9c35, server=828b3f090c91,40359,1732657789839}] 2024-11-26T21:49:56,394 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure a2d6b8d1f56582e3f52e3cfe7a116c1d, server=828b3f090c91,40359,1732657789839}] 2024-11-26T21:49:56,398 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c4145a9c4106377623484a3eb786fe8a, server=828b3f090c91,40359,1732657789839}] 2024-11-26T21:49:56,421 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(132): Open test,eee,1732657792023.651cf53d9624e6cc98e46d6578c71496. 2024-11-26T21:49:56,421 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7752): Opening region: {ENCODED => 651cf53d9624e6cc98e46d6578c71496, NAME => 'test,eee,1732657792023.651cf53d9624e6cc98e46d6578c71496.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-26T21:49:56,422 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 651cf53d9624e6cc98e46d6578c71496 2024-11-26T21:49:56,422 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(898): Instantiated test,eee,1732657792023.651cf53d9624e6cc98e46d6578c71496.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:56,422 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7794): checking encryption for 651cf53d9624e6cc98e46d6578c71496 2024-11-26T21:49:56,422 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7797): checking classloading for 651cf53d9624e6cc98e46d6578c71496 2024-11-26T21:49:56,423 INFO [StoreOpener-651cf53d9624e6cc98e46d6578c71496-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 651cf53d9624e6cc98e46d6578c71496 2024-11-26T21:49:56,424 INFO [StoreOpener-651cf53d9624e6cc98e46d6578c71496-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 651cf53d9624e6cc98e46d6578c71496 columnFamilyName family 2024-11-26T21:49:56,425 DEBUG [StoreOpener-651cf53d9624e6cc98e46d6578c71496-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:56,425 INFO [StoreOpener-651cf53d9624e6cc98e46d6578c71496-1 {}] regionserver.HStore(327): Store=651cf53d9624e6cc98e46d6578c71496/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:49:56,425 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1038): replaying wal for 651cf53d9624e6cc98e46d6578c71496 2024-11-26T21:49:56,426 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/651cf53d9624e6cc98e46d6578c71496 2024-11-26T21:49:56,428 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/651cf53d9624e6cc98e46d6578c71496 2024-11-26T21:49:56,429 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1048): stopping wal replay for 651cf53d9624e6cc98e46d6578c71496 2024-11-26T21:49:56,429 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1060): Cleaning up temporary data for 651cf53d9624e6cc98e46d6578c71496 2024-11-26T21:49:56,431 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1093): writing seq id for 651cf53d9624e6cc98e46d6578c71496 2024-11-26T21:49:56,432 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1114): Opened 651cf53d9624e6cc98e46d6578c71496; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59082555, jitterRate=-0.11960132420063019}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T21:49:56,433 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 651cf53d9624e6cc98e46d6578c71496 2024-11-26T21:49:56,433 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1006): Region open journal for 651cf53d9624e6cc98e46d6578c71496: Running coprocessor pre-open hook at 1732657796422Writing region info on filesystem at 1732657796422Initializing all the Stores at 1732657796423 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657796423Cleaning up temporary data from old regions at 1732657796429 (+6 ms)Running coprocessor post-open hooks at 1732657796433 (+4 ms)Region opened successfully at 1732657796433 2024-11-26T21:49:56,434 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1732657792023.651cf53d9624e6cc98e46d6578c71496., pid=72, masterSystemTime=1732657796416 2024-11-26T21:49:56,437 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1732657792023.651cf53d9624e6cc98e46d6578c71496. 2024-11-26T21:49:56,437 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(153): Opened test,eee,1732657792023.651cf53d9624e6cc98e46d6578c71496. 2024-11-26T21:49:56,437 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,sss,1732657792023.2b8b8290cc9d27b8d226e5dc2b7d770d. 2024-11-26T21:49:56,437 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => 2b8b8290cc9d27b8d226e5dc2b7d770d, NAME => 'test,sss,1732657792023.2b8b8290cc9d27b8d226e5dc2b7d770d.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-26T21:49:56,438 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2b8b8290cc9d27b8d226e5dc2b7d770d 2024-11-26T21:49:56,438 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,sss,1732657792023.2b8b8290cc9d27b8d226e5dc2b7d770d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:56,438 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for 2b8b8290cc9d27b8d226e5dc2b7d770d 2024-11-26T21:49:56,438 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for 2b8b8290cc9d27b8d226e5dc2b7d770d 2024-11-26T21:49:56,439 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-11-26T21:49:56,439 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,440 INFO [StoreOpener-2b8b8290cc9d27b8d226e5dc2b7d770d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2b8b8290cc9d27b8d226e5dc2b7d770d 2024-11-26T21:49:56,439 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,442 INFO [StoreOpener-2b8b8290cc9d27b8d226e5dc2b7d770d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b8b8290cc9d27b8d226e5dc2b7d770d columnFamilyName family 2024-11-26T21:49:56,442 DEBUG [StoreOpener-2b8b8290cc9d27b8d226e5dc2b7d770d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:56,443 INFO [StoreOpener-2b8b8290cc9d27b8d226e5dc2b7d770d-1 {}] regionserver.HStore(327): Store=2b8b8290cc9d27b8d226e5dc2b7d770d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:49:56,443 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for 2b8b8290cc9d27b8d226e5dc2b7d770d 2024-11-26T21:49:56,444 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/2b8b8290cc9d27b8d226e5dc2b7d770d 2024-11-26T21:49:56,446 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/2b8b8290cc9d27b8d226e5dc2b7d770d 2024-11-26T21:49:56,447 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for 2b8b8290cc9d27b8d226e5dc2b7d770d 2024-11-26T21:49:56,447 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for 2b8b8290cc9d27b8d226e5dc2b7d770d 2024-11-26T21:49:56,450 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for 2b8b8290cc9d27b8d226e5dc2b7d770d 2024-11-26T21:49:56,451 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened 2b8b8290cc9d27b8d226e5dc2b7d770d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73341053, jitterRate=0.09286685287952423}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T21:49:56,451 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2b8b8290cc9d27b8d226e5dc2b7d770d 2024-11-26T21:49:56,451 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for 2b8b8290cc9d27b8d226e5dc2b7d770d: Running coprocessor pre-open hook at 1732657796438Writing region info on filesystem at 1732657796438Initializing all the Stores at 1732657796440 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657796440Cleaning up temporary data from old regions at 1732657796447 (+7 ms)Running coprocessor post-open hooks at 1732657796451 (+4 ms)Region opened successfully at 1732657796451 2024-11-26T21:49:56,450 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 4 more 2024-11-26T21:49:56,452 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T21:49:56,452 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c4145a9c4106377623484a3eb786fe8a, server=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,452 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:56,453 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c4145a9c4106377623484a3eb786fe8a, server=828b3f090c91,40359,1732657789839 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,453 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1732657792023.2b8b8290cc9d27b8d226e5dc2b7d770d., pid=76, masterSystemTime=1732657796416 2024-11-26T21:49:56,456 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1732657792023.2b8b8290cc9d27b8d226e5dc2b7d770d. 2024-11-26T21:49:56,456 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened test,sss,1732657792023.2b8b8290cc9d27b8d226e5dc2b7d770d. 2024-11-26T21:49:56,456 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,kkk,1732657792023.334949367d92cc18a393737b69358f37. 2024-11-26T21:49:56,456 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => 334949367d92cc18a393737b69358f37, NAME => 'test,kkk,1732657792023.334949367d92cc18a393737b69358f37.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-26T21:49:56,456 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 334949367d92cc18a393737b69358f37 2024-11-26T21:49:56,456 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,kkk,1732657792023.334949367d92cc18a393737b69358f37.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:56,456 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for 334949367d92cc18a393737b69358f37 2024-11-26T21:49:56,456 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for 334949367d92cc18a393737b69358f37 2024-11-26T21:49:56,458 INFO [StoreOpener-334949367d92cc18a393737b69358f37-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 334949367d92cc18a393737b69358f37 2024-11-26T21:49:56,459 INFO [StoreOpener-334949367d92cc18a393737b69358f37-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 334949367d92cc18a393737b69358f37 columnFamilyName family 2024-11-26T21:49:56,459 DEBUG [StoreOpener-334949367d92cc18a393737b69358f37-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:56,460 INFO [StoreOpener-334949367d92cc18a393737b69358f37-1 {}] regionserver.HStore(327): Store=334949367d92cc18a393737b69358f37/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:49:56,460 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for 334949367d92cc18a393737b69358f37 2024-11-26T21:49:56,461 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/334949367d92cc18a393737b69358f37 2024-11-26T21:49:56,463 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/334949367d92cc18a393737b69358f37 2024-11-26T21:49:56,463 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for 334949367d92cc18a393737b69358f37 2024-11-26T21:49:56,463 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for 334949367d92cc18a393737b69358f37 2024-11-26T21:49:56,465 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for 334949367d92cc18a393737b69358f37 2024-11-26T21:49:56,466 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened 334949367d92cc18a393737b69358f37; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59639214, jitterRate=-0.11130645871162415}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T21:49:56,467 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 334949367d92cc18a393737b69358f37 2024-11-26T21:49:56,467 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for 334949367d92cc18a393737b69358f37: Running coprocessor pre-open hook at 1732657796457Writing region info on filesystem at 1732657796457Initializing all the Stores at 1732657796458 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657796458Cleaning up temporary data from old regions at 1732657796463 (+5 ms)Running coprocessor post-open hooks at 1732657796467 (+4 ms)Region opened successfully at 1732657796467 2024-11-26T21:49:56,468 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1732657792023.334949367d92cc18a393737b69358f37., pid=75, masterSystemTime=1732657796416 2024-11-26T21:49:56,471 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1732657792023.334949367d92cc18a393737b69358f37. 2024-11-26T21:49:56,471 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,kkk,1732657792023.334949367d92cc18a393737b69358f37. 2024-11-26T21:49:56,472 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(132): Open test,yyy,1732657792023.1e7efa0e8c09fd1f0e1a765d7fc38181. 2024-11-26T21:49:56,472 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7752): Opening region: {ENCODED => 1e7efa0e8c09fd1f0e1a765d7fc38181, NAME => 'test,yyy,1732657792023.1e7efa0e8c09fd1f0e1a765d7fc38181.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-26T21:49:56,472 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1e7efa0e8c09fd1f0e1a765d7fc38181 2024-11-26T21:49:56,472 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(898): Instantiated test,yyy,1732657792023.1e7efa0e8c09fd1f0e1a765d7fc38181.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:56,473 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7794): checking encryption for 1e7efa0e8c09fd1f0e1a765d7fc38181 2024-11-26T21:49:56,473 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7797): checking classloading for 1e7efa0e8c09fd1f0e1a765d7fc38181 2024-11-26T21:49:56,475 INFO [StoreOpener-1e7efa0e8c09fd1f0e1a765d7fc38181-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1e7efa0e8c09fd1f0e1a765d7fc38181 2024-11-26T21:49:56,476 INFO [StoreOpener-1e7efa0e8c09fd1f0e1a765d7fc38181-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e7efa0e8c09fd1f0e1a765d7fc38181 columnFamilyName family 2024-11-26T21:49:56,476 DEBUG [StoreOpener-1e7efa0e8c09fd1f0e1a765d7fc38181-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:56,477 INFO [StoreOpener-1e7efa0e8c09fd1f0e1a765d7fc38181-1 {}] regionserver.HStore(327): Store=1e7efa0e8c09fd1f0e1a765d7fc38181/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:49:56,477 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1038): replaying wal for 1e7efa0e8c09fd1f0e1a765d7fc38181 2024-11-26T21:49:56,478 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/1e7efa0e8c09fd1f0e1a765d7fc38181 2024-11-26T21:49:56,480 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/1e7efa0e8c09fd1f0e1a765d7fc38181 2024-11-26T21:49:56,481 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1048): stopping wal replay for 1e7efa0e8c09fd1f0e1a765d7fc38181 2024-11-26T21:49:56,481 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1060): Cleaning up temporary data for 1e7efa0e8c09fd1f0e1a765d7fc38181 2024-11-26T21:49:56,484 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1093): writing seq id for 1e7efa0e8c09fd1f0e1a765d7fc38181 2024-11-26T21:49:56,485 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1114): Opened 1e7efa0e8c09fd1f0e1a765d7fc38181; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72585497, jitterRate=0.08160819113254547}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T21:49:56,485 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1e7efa0e8c09fd1f0e1a765d7fc38181 2024-11-26T21:49:56,485 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1006): Region open journal for 1e7efa0e8c09fd1f0e1a765d7fc38181: Running coprocessor pre-open hook at 1732657796473Writing region info on filesystem at 1732657796473Initializing all the Stores at 1732657796474 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657796474Cleaning up temporary data from old regions at 1732657796481 (+7 ms)Running coprocessor post-open hooks at 1732657796485 (+4 ms)Region opened successfully at 1732657796485 2024-11-26T21:49:56,486 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1732657792023.1e7efa0e8c09fd1f0e1a765d7fc38181., pid=74, masterSystemTime=1732657796416 2024-11-26T21:49:56,488 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1732657792023.1e7efa0e8c09fd1f0e1a765d7fc38181. 2024-11-26T21:49:56,489 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(153): Opened test,yyy,1732657792023.1e7efa0e8c09fd1f0e1a765d7fc38181. 2024-11-26T21:49:56,522 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T21:49:56,522 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:56,522 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T21:49:56,522 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T21:49:56,523 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T21:49:56,523 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T21:49:56,523 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T21:49:56,524 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T21:49:56,524 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T21:49:56,532 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,,1732657792023.bf200defbfcf83538e813f9fa91d9c35. 2024-11-26T21:49:56,532 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => bf200defbfcf83538e813f9fa91d9c35, NAME => 'test,,1732657792023.bf200defbfcf83538e813f9fa91d9c35.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-26T21:49:56,532 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test bf200defbfcf83538e813f9fa91d9c35 2024-11-26T21:49:56,533 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,,1732657792023.bf200defbfcf83538e813f9fa91d9c35.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:49:56,533 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T21:49:56,533 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for bf200defbfcf83538e813f9fa91d9c35 2024-11-26T21:49:56,533 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for bf200defbfcf83538e813f9fa91d9c35 2024-11-26T21:49:56,533 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,533 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,533 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,533 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,533 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,533 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,533 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,534 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,534 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,534 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,534 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,534 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,534 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,534 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,534 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,534 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,534 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T21:49:56,534 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T21:49:56,534 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,534 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,534 INFO [StoreOpener-bf200defbfcf83538e813f9fa91d9c35-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region bf200defbfcf83538e813f9fa91d9c35 2024-11-26T21:49:56,534 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T21:49:56,534 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T21:49:56,535 INFO [StoreOpener-bf200defbfcf83538e813f9fa91d9c35-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf200defbfcf83538e813f9fa91d9c35 columnFamilyName family 2024-11-26T21:49:56,535 DEBUG [StoreOpener-bf200defbfcf83538e813f9fa91d9c35-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:49:56,535 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T21:49:56,536 INFO [StoreOpener-bf200defbfcf83538e813f9fa91d9c35-1 {}] regionserver.HStore(327): Store=bf200defbfcf83538e813f9fa91d9c35/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:49:56,536 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for bf200defbfcf83538e813f9fa91d9c35 2024-11-26T21:49:56,537 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/bf200defbfcf83538e813f9fa91d9c35 2024-11-26T21:49:56,537 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T21:49:56,538 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/bf200defbfcf83538e813f9fa91d9c35 2024-11-26T21:49:56,539 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for bf200defbfcf83538e813f9fa91d9c35 2024-11-26T21:49:56,539 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for bf200defbfcf83538e813f9fa91d9c35 2024-11-26T21:49:56,541 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for bf200defbfcf83538e813f9fa91d9c35 2024-11-26T21:49:56,542 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened bf200defbfcf83538e813f9fa91d9c35; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70612917, jitterRate=0.05221445858478546}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T21:49:56,542 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bf200defbfcf83538e813f9fa91d9c35 2024-11-26T21:49:56,542 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for bf200defbfcf83538e813f9fa91d9c35: Running coprocessor pre-open hook at 1732657796533Writing region info on filesystem at 1732657796533Initializing all the Stores at 1732657796534 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657796534Cleaning up temporary data from old regions at 1732657796539 (+5 ms)Running coprocessor post-open hooks at 1732657796542 (+3 ms)Region opened successfully at 1732657796542 2024-11-26T21:49:56,543 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1732657792023.bf200defbfcf83538e813f9fa91d9c35., pid=77, masterSystemTime=1732657796527 2024-11-26T21:49:56,544 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44203 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,544 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true 2024-11-26T21:49:56,544 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,544 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,544 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T21:49:56,545 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,545 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,545 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,545 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN 2024-11-26T21:49:56,545 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN 2024-11-26T21:49:56,545 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN 2024-11-26T21:49:56,545 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN 2024-11-26T21:49:56,545 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN 2024-11-26T21:49:56,545 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN 2024-11-26T21:49:56,545 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN 2024-11-26T21:49:56,545 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN 2024-11-26T21:49:56,545 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,545 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21af909e3b8cf553daecee4e59c0898b, server=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,545 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e7efa0e8c09fd1f0e1a765d7fc38181, server=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,545 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 334949367d92cc18a393737b69358f37, server=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,545 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b8b8290cc9d27b8d226e5dc2b7d770d, server=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,545 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure bf200defbfcf83538e813f9fa91d9c35, server=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,546 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure a2d6b8d1f56582e3f52e3cfe7a116c1d, server=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,546 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c4145a9c4106377623484a3eb786fe8a, server=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,546 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,546 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,546 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,546 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,546 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,546 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,546 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,546 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,546 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN 2024-11-26T21:49:56,546 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,546 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN 2024-11-26T21:49:56,546 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,546 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN 2024-11-26T21:49:56,546 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,547 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN 2024-11-26T21:49:56,547 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,547 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN 2024-11-26T21:49:56,547 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,547 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN 2024-11-26T21:49:56,547 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,547 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN 2024-11-26T21:49:56,547 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,547 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN 2024-11-26T21:49:56,546 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "828b3f090c91" port: 40359 start_code: 1732657789839 } transition { transition_code: OPENED region_info { region_id: 1732657792023 table_name { namespace: "default" qualifier: "test" } start_key: "" end_key: "bbb" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 77 initiating_master_active_time: 1732657789961 }; retry (#0) after 1000ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-26T21:49:56,547 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,547 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21af909e3b8cf553daecee4e59c0898b, server=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,547 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e7efa0e8c09fd1f0e1a765d7fc38181, server=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,547 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 334949367d92cc18a393737b69358f37, server=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,547 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b8b8290cc9d27b8d226e5dc2b7d770d, server=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,547 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure bf200defbfcf83538e813f9fa91d9c35, server=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,547 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure a2d6b8d1f56582e3f52e3cfe7a116c1d, server=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,547 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c4145a9c4106377623484a3eb786fe8a, server=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,548 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 23 msec 2024-11-26T21:49:56,550 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=61 2024-11-26T21:49:56,550 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=828b3f090c91,40359,1732657789839, table=hbase:meta, region=1588230740 2024-11-26T21:49:56,550 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 828b3f090c91,40359,1732657789839, state=OPEN 2024-11-26T21:49:56,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,552 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,552 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,552 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,554 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN to state=OFFLINE, location=null, table=test, region=c4145a9c4106377623484a3eb786fe8a to restore RIT 2024-11-26T21:49:56,554 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN to state=OFFLINE, location=null, table=test, region=334949367d92cc18a393737b69358f37 to restore RIT 2024-11-26T21:49:56,554 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN to state=OFFLINE, location=null, table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d to restore RIT 2024-11-26T21:49:56,554 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN to state=OFFLINE, location=null, table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181 to restore RIT 2024-11-26T21:49:56,554 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN to state=OFFLINE, location=null, table=test, region=21af909e3b8cf553daecee4e59c0898b to restore RIT 2024-11-26T21:49:56,554 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN to state=OFFLINE, location=null, table=test, region=bf200defbfcf83538e813f9fa91d9c35 to restore RIT 2024-11-26T21:49:56,554 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN to state=OFFLINE, location=null, table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d to restore RIT 2024-11-26T21:49:56,554 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN to state=OFFLINE, location=null, table=test, region=651cf53d9624e6cc98e46d6578c71496 to restore RIT 2024-11-26T21:49:56,555 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732657826555 2024-11-26T21:49:56,555 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T21:49:56,555 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,557 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T21:49:56,558 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T21:49:56,559 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure a2d6b8d1f56582e3f52e3cfe7a116c1d, server=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,559 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:56,559 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure a2d6b8d1f56582e3f52e3cfe7a116c1d, server=828b3f090c91,40359,1732657789839 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,583 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf200defbfcf83538e813f9fa91d9c35, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36699c847bc0e6c6b06fd8533d3c9edb, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34e003c01eb3f0a2e10e33152402164c, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a7c374815f888db179beadaf2489ad7, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=651cf53d9624e6cc98e46d6578c71496, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0a1ab633a90e6b5f5e9ba8ce6f66010, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8be22ac0f4cc7a17874b9d10ba204518, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a2d6b8d1f56582e3f52e3cfe7a116c1d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d22f8ae31d7e7a4d0c284e3b9676a393, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdd2d12bbd55678c7f638b4ef0fde3da, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=334949367d92cc18a393737b69358f37, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac402503c294574c0e555c45a12f0a4e, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06a1142ff2ac98bd39e6f73547d6aa98, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4145a9c4106377623484a3eb786fe8a, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a6bb8489f79432157c419605ac84965, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c5d2bc20a7b33de1c8b5a53f7fee57, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=996589dbb2419ea602bc5e348cbd05df, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cf2d9c6c6cf36bd61bb35f51df8f42, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b8b8290cc9d27b8d226e5dc2b7d770d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ef98738dc2598149d89e210ccdb4e6f, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=317db83a078d27fdfa108a4454669013, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21af909e3b8cf553daecee4e59c0898b, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f45cc001ffd649bcfbfe0f7af1630961, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e9187292024abc7067d470edad3ae67, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e7efa0e8c09fd1f0e1a765d7fc38181, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1062179709b27df36edcb7f6b054a4a6, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,588 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T21:49:56,589 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732657856589 2024-11-26T21:49:56,589 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732657916589 2024-11-26T21:49:56,589 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 33 msec 2024-11-26T21:49:56,589 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T21:49:56,589 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:56,589 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T21:49:56,589 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T21:49:56,589 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T21:49:56,589 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T21:49:56,589 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T21:49:56,590 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T21:49:56,591 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T21:49:56,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T21:49:56,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T21:49:56,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T21:49:56,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T21:49:56,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T21:49:56,596 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T21:49:56,597 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T21:49:56,600 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true 2024-11-26T21:49:56,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T21:49:56,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN 2024-11-26T21:49:56,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN 2024-11-26T21:49:56,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN 2024-11-26T21:49:56,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN 2024-11-26T21:49:56,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN 2024-11-26T21:49:56,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN 2024-11-26T21:49:56,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN 2024-11-26T21:49:56,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN 2024-11-26T21:49:56,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21af909e3b8cf553daecee4e59c0898b, server=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e7efa0e8c09fd1f0e1a765d7fc38181, server=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 334949367d92cc18a393737b69358f37, server=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b8b8290cc9d27b8d226e5dc2b7d770d, server=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure bf200defbfcf83538e813f9fa91d9c35, server=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure a2d6b8d1f56582e3f52e3cfe7a116c1d, server=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,602 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,602 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,602 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,602 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,602 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,602 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,602 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,602 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,602 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN 2024-11-26T21:49:56,602 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,602 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN 2024-11-26T21:49:56,602 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,602 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN 2024-11-26T21:49:56,602 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,602 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN 2024-11-26T21:49:56,602 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,602 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN 2024-11-26T21:49:56,602 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,602 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN 2024-11-26T21:49:56,602 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,602 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN 2024-11-26T21:49:56,602 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,602 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN 2024-11-26T21:49:56,602 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,602 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21af909e3b8cf553daecee4e59c0898b, server=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,602 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e7efa0e8c09fd1f0e1a765d7fc38181, server=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,602 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 334949367d92cc18a393737b69358f37, server=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,603 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b8b8290cc9d27b8d226e5dc2b7d770d, server=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,603 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure bf200defbfcf83538e813f9fa91d9c35, server=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,603 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure a2d6b8d1f56582e3f52e3cfe7a116c1d, server=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,603 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-11-26T21:49:56,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=61 2024-11-26T21:49:56,604 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=828b3f090c91,40359,1732657789839, table=hbase:meta, region=1588230740 2024-11-26T21:49:56,604 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 828b3f090c91,40359,1732657789839, state=OPEN 2024-11-26T21:49:56,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,606 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,606 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,606 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,608 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN to state=OFFLINE, location=null, table=test, region=c4145a9c4106377623484a3eb786fe8a to restore RIT 2024-11-26T21:49:56,608 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN to state=OFFLINE, location=null, table=test, region=334949367d92cc18a393737b69358f37 to restore RIT 2024-11-26T21:49:56,608 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN to state=OFFLINE, location=null, table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d to restore RIT 2024-11-26T21:49:56,608 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN to state=OFFLINE, location=null, table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181 to restore RIT 2024-11-26T21:49:56,608 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN to state=OFFLINE, location=null, table=test, region=21af909e3b8cf553daecee4e59c0898b to restore RIT 2024-11-26T21:49:56,608 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN to state=OFFLINE, location=null, table=test, region=bf200defbfcf83538e813f9fa91d9c35 to restore RIT 2024-11-26T21:49:56,608 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN to state=OFFLINE, location=null, table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d to restore RIT 2024-11-26T21:49:56,608 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN to state=OFFLINE, location=null, table=test, region=651cf53d9624e6cc98e46d6578c71496 to restore RIT 2024-11-26T21:49:56,609 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732657826609 2024-11-26T21:49:56,609 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T21:49:56,609 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,611 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T21:49:56,612 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T21:49:56,612 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure bf200defbfcf83538e813f9fa91d9c35, server=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,612 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:56,612 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure bf200defbfcf83538e813f9fa91d9c35, server=828b3f090c91,40359,1732657789839 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf200defbfcf83538e813f9fa91d9c35, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36699c847bc0e6c6b06fd8533d3c9edb, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34e003c01eb3f0a2e10e33152402164c, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a7c374815f888db179beadaf2489ad7, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=651cf53d9624e6cc98e46d6578c71496, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0a1ab633a90e6b5f5e9ba8ce6f66010, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8be22ac0f4cc7a17874b9d10ba204518, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a2d6b8d1f56582e3f52e3cfe7a116c1d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d22f8ae31d7e7a4d0c284e3b9676a393, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdd2d12bbd55678c7f638b4ef0fde3da, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=334949367d92cc18a393737b69358f37, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac402503c294574c0e555c45a12f0a4e, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06a1142ff2ac98bd39e6f73547d6aa98, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4145a9c4106377623484a3eb786fe8a, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a6bb8489f79432157c419605ac84965, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c5d2bc20a7b33de1c8b5a53f7fee57, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=996589dbb2419ea602bc5e348cbd05df, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cf2d9c6c6cf36bd61bb35f51df8f42, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b8b8290cc9d27b8d226e5dc2b7d770d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,622 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ef98738dc2598149d89e210ccdb4e6f, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,622 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=317db83a078d27fdfa108a4454669013, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,622 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21af909e3b8cf553daecee4e59c0898b, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,622 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f45cc001ffd649bcfbfe0f7af1630961, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,622 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e9187292024abc7067d470edad3ae67, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,622 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e7efa0e8c09fd1f0e1a765d7fc38181, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,623 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1062179709b27df36edcb7f6b054a4a6, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,623 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T21:49:56,623 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732657856623 2024-11-26T21:49:56,623 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732657916623 2024-11-26T21:49:56,623 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 13 msec 2024-11-26T21:49:56,623 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T21:49:56,623 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:56,623 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T21:49:56,623 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T21:49:56,623 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T21:49:56,623 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T21:49:56,623 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T21:49:56,624 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T21:49:56,624 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T21:49:56,628 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T21:49:56,628 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,628 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T21:49:56,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T21:49:56,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T21:49:56,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T21:49:56,629 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T21:49:56,630 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T21:49:56,633 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true 2024-11-26T21:49:56,633 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,633 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,633 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T21:49:56,633 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,633 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,633 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,633 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN 2024-11-26T21:49:56,633 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN 2024-11-26T21:49:56,633 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN 2024-11-26T21:49:56,633 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN 2024-11-26T21:49:56,633 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN 2024-11-26T21:49:56,633 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN 2024-11-26T21:49:56,633 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN 2024-11-26T21:49:56,633 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN 2024-11-26T21:49:56,633 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,633 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21af909e3b8cf553daecee4e59c0898b, server=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,633 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e7efa0e8c09fd1f0e1a765d7fc38181, server=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,633 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 334949367d92cc18a393737b69358f37, server=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,633 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b8b8290cc9d27b8d226e5dc2b7d770d, server=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,633 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure bf200defbfcf83538e813f9fa91d9c35, server=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,633 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,633 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,633 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,633 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,633 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,634 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,634 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,634 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,634 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN 2024-11-26T21:49:56,634 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,634 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN 2024-11-26T21:49:56,634 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,634 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN 2024-11-26T21:49:56,634 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,634 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN 2024-11-26T21:49:56,634 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,634 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN 2024-11-26T21:49:56,634 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,634 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN 2024-11-26T21:49:56,634 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,634 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN 2024-11-26T21:49:56,634 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,634 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN 2024-11-26T21:49:56,634 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,634 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21af909e3b8cf553daecee4e59c0898b, server=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,634 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e7efa0e8c09fd1f0e1a765d7fc38181, server=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,634 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 334949367d92cc18a393737b69358f37, server=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,634 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b8b8290cc9d27b8d226e5dc2b7d770d, server=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,634 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure bf200defbfcf83538e813f9fa91d9c35, server=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,635 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-26T21:49:56,636 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=61 2024-11-26T21:49:56,636 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=828b3f090c91,40359,1732657789839, table=hbase:meta, region=1588230740 2024-11-26T21:49:56,636 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 828b3f090c91,40359,1732657789839, state=OPEN 2024-11-26T21:49:56,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,637 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,637 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,637 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,638 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN to state=OFFLINE, location=null, table=test, region=c4145a9c4106377623484a3eb786fe8a to restore RIT 2024-11-26T21:49:56,638 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN to state=OFFLINE, location=null, table=test, region=334949367d92cc18a393737b69358f37 to restore RIT 2024-11-26T21:49:56,638 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN to state=OFFLINE, location=null, table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d to restore RIT 2024-11-26T21:49:56,638 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN to state=OFFLINE, location=null, table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181 to restore RIT 2024-11-26T21:49:56,638 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN to state=OFFLINE, location=null, table=test, region=21af909e3b8cf553daecee4e59c0898b to restore RIT 2024-11-26T21:49:56,638 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN to state=OFFLINE, location=null, table=test, region=bf200defbfcf83538e813f9fa91d9c35 to restore RIT 2024-11-26T21:49:56,638 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN to state=OFFLINE, location=null, table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d to restore RIT 2024-11-26T21:49:56,638 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN to state=OFFLINE, location=null, table=test, region=651cf53d9624e6cc98e46d6578c71496 to restore RIT 2024-11-26T21:49:56,639 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732657826639 2024-11-26T21:49:56,639 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T21:49:56,639 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,640 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T21:49:56,642 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T21:49:56,642 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b8b8290cc9d27b8d226e5dc2b7d770d, server=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,642 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:56,642 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b8b8290cc9d27b8d226e5dc2b7d770d, server=828b3f090c91,42273,1732657789877 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,647 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf200defbfcf83538e813f9fa91d9c35, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,647 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36699c847bc0e6c6b06fd8533d3c9edb, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,647 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34e003c01eb3f0a2e10e33152402164c, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,648 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a7c374815f888db179beadaf2489ad7, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,648 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=651cf53d9624e6cc98e46d6578c71496, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,648 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0a1ab633a90e6b5f5e9ba8ce6f66010, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,648 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8be22ac0f4cc7a17874b9d10ba204518, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,648 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a2d6b8d1f56582e3f52e3cfe7a116c1d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,648 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d22f8ae31d7e7a4d0c284e3b9676a393, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,649 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdd2d12bbd55678c7f638b4ef0fde3da, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,649 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=334949367d92cc18a393737b69358f37, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,649 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac402503c294574c0e555c45a12f0a4e, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,649 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06a1142ff2ac98bd39e6f73547d6aa98, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,649 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4145a9c4106377623484a3eb786fe8a, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,649 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a6bb8489f79432157c419605ac84965, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,649 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c5d2bc20a7b33de1c8b5a53f7fee57, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,650 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=996589dbb2419ea602bc5e348cbd05df, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,650 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cf2d9c6c6cf36bd61bb35f51df8f42, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,650 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b8b8290cc9d27b8d226e5dc2b7d770d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,650 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ef98738dc2598149d89e210ccdb4e6f, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,651 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=317db83a078d27fdfa108a4454669013, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,651 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21af909e3b8cf553daecee4e59c0898b, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,651 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f45cc001ffd649bcfbfe0f7af1630961, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,651 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e9187292024abc7067d470edad3ae67, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,651 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e7efa0e8c09fd1f0e1a765d7fc38181, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,651 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1062179709b27df36edcb7f6b054a4a6, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,651 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T21:49:56,651 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732657856651 2024-11-26T21:49:56,652 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732657916651 2024-11-26T21:49:56,652 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-11-26T21:49:56,652 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T21:49:56,652 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:56,652 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T21:49:56,652 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T21:49:56,652 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T21:49:56,652 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T21:49:56,652 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T21:49:56,652 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T21:49:56,653 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T21:49:56,657 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T21:49:56,657 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,657 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,657 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,657 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,657 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,657 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,657 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,657 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,657 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,657 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,657 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,657 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,657 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,657 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T21:49:56,657 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T21:49:56,657 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,657 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,657 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T21:49:56,657 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T21:49:56,657 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T21:49:56,658 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T21:49:56,661 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true 2024-11-26T21:49:56,661 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,661 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,661 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T21:49:56,661 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,661 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,661 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,661 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN 2024-11-26T21:49:56,661 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN 2024-11-26T21:49:56,661 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN 2024-11-26T21:49:56,661 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN 2024-11-26T21:49:56,661 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN 2024-11-26T21:49:56,661 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN 2024-11-26T21:49:56,661 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN 2024-11-26T21:49:56,661 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN 2024-11-26T21:49:56,661 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,661 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21af909e3b8cf553daecee4e59c0898b, server=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,661 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e7efa0e8c09fd1f0e1a765d7fc38181, server=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,661 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 334949367d92cc18a393737b69358f37, server=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,661 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b8b8290cc9d27b8d226e5dc2b7d770d, server=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,661 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,662 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,662 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,662 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,662 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,662 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,662 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,662 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,662 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN 2024-11-26T21:49:56,662 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,662 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN 2024-11-26T21:49:56,662 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,662 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN 2024-11-26T21:49:56,662 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,662 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN 2024-11-26T21:49:56,662 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,662 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN 2024-11-26T21:49:56,662 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,662 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN 2024-11-26T21:49:56,662 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,662 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN 2024-11-26T21:49:56,662 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,662 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN 2024-11-26T21:49:56,662 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,662 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21af909e3b8cf553daecee4e59c0898b, server=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,662 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e7efa0e8c09fd1f0e1a765d7fc38181, server=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,662 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 334949367d92cc18a393737b69358f37, server=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,662 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b8b8290cc9d27b8d226e5dc2b7d770d, server=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,663 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-26T21:49:56,664 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=61 2024-11-26T21:49:56,664 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=828b3f090c91,40359,1732657789839, table=hbase:meta, region=1588230740 2024-11-26T21:49:56,664 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 828b3f090c91,40359,1732657789839, state=OPEN 2024-11-26T21:49:56,665 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,665 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,665 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,665 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,665 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,665 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,666 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN to state=OFFLINE, location=null, table=test, region=c4145a9c4106377623484a3eb786fe8a to restore RIT 2024-11-26T21:49:56,666 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN to state=OFFLINE, location=null, table=test, region=334949367d92cc18a393737b69358f37 to restore RIT 2024-11-26T21:49:56,666 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN to state=OFFLINE, location=null, table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d to restore RIT 2024-11-26T21:49:56,666 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN to state=OFFLINE, location=null, table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181 to restore RIT 2024-11-26T21:49:56,666 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN to state=OFFLINE, location=null, table=test, region=21af909e3b8cf553daecee4e59c0898b to restore RIT 2024-11-26T21:49:56,666 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN to state=OFFLINE, location=null, table=test, region=bf200defbfcf83538e813f9fa91d9c35 to restore RIT 2024-11-26T21:49:56,666 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN to state=OFFLINE, location=null, table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d to restore RIT 2024-11-26T21:49:56,667 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN to state=OFFLINE, location=null, table=test, region=651cf53d9624e6cc98e46d6578c71496 to restore RIT 2024-11-26T21:49:56,667 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732657826667 2024-11-26T21:49:56,667 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T21:49:56,667 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,668 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T21:49:56,670 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T21:49:56,670 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 334949367d92cc18a393737b69358f37, server=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,670 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:56,670 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 334949367d92cc18a393737b69358f37, server=828b3f090c91,42273,1732657789877 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,674 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf200defbfcf83538e813f9fa91d9c35, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,674 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36699c847bc0e6c6b06fd8533d3c9edb, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,674 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34e003c01eb3f0a2e10e33152402164c, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,675 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a7c374815f888db179beadaf2489ad7, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,675 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=651cf53d9624e6cc98e46d6578c71496, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,675 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0a1ab633a90e6b5f5e9ba8ce6f66010, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,675 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8be22ac0f4cc7a17874b9d10ba204518, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,675 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a2d6b8d1f56582e3f52e3cfe7a116c1d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,676 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d22f8ae31d7e7a4d0c284e3b9676a393, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,676 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdd2d12bbd55678c7f638b4ef0fde3da, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,676 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=334949367d92cc18a393737b69358f37, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,676 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac402503c294574c0e555c45a12f0a4e, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,676 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06a1142ff2ac98bd39e6f73547d6aa98, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,676 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4145a9c4106377623484a3eb786fe8a, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,676 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a6bb8489f79432157c419605ac84965, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,677 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c5d2bc20a7b33de1c8b5a53f7fee57, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,677 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=996589dbb2419ea602bc5e348cbd05df, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,677 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cf2d9c6c6cf36bd61bb35f51df8f42, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,677 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b8b8290cc9d27b8d226e5dc2b7d770d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,677 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ef98738dc2598149d89e210ccdb4e6f, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,677 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=317db83a078d27fdfa108a4454669013, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,677 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21af909e3b8cf553daecee4e59c0898b, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f45cc001ffd649bcfbfe0f7af1630961, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e9187292024abc7067d470edad3ae67, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e7efa0e8c09fd1f0e1a765d7fc38181, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1062179709b27df36edcb7f6b054a4a6, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,678 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T21:49:56,678 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732657856678 2024-11-26T21:49:56,678 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732657916678 2024-11-26T21:49:56,678 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-11-26T21:49:56,678 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T21:49:56,678 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:56,678 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T21:49:56,678 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T21:49:56,679 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T21:49:56,679 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T21:49:56,679 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T21:49:56,679 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T21:49:56,680 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T21:49:56,683 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T21:49:56,683 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,683 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,683 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,683 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,683 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,683 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,683 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,683 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,683 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,684 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,684 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,684 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,684 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T21:49:56,684 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T21:49:56,684 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,684 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,684 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T21:49:56,684 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T21:49:56,684 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T21:49:56,684 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44203 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,685 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T21:49:56,687 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44203 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true 2024-11-26T21:49:56,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T21:49:56,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN 2024-11-26T21:49:56,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN 2024-11-26T21:49:56,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN 2024-11-26T21:49:56,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN 2024-11-26T21:49:56,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN 2024-11-26T21:49:56,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN 2024-11-26T21:49:56,689 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN 2024-11-26T21:49:56,689 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN 2024-11-26T21:49:56,689 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,689 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21af909e3b8cf553daecee4e59c0898b, server=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,689 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e7efa0e8c09fd1f0e1a765d7fc38181, server=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,689 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 334949367d92cc18a393737b69358f37, server=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,689 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,689 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,689 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,689 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,689 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,689 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,689 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,689 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,689 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN 2024-11-26T21:49:56,689 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,689 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN 2024-11-26T21:49:56,689 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,689 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN 2024-11-26T21:49:56,689 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,689 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN 2024-11-26T21:49:56,689 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,689 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN 2024-11-26T21:49:56,689 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,689 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN 2024-11-26T21:49:56,689 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,689 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN 2024-11-26T21:49:56,689 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,690 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN 2024-11-26T21:49:56,690 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,690 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21af909e3b8cf553daecee4e59c0898b, server=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,690 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e7efa0e8c09fd1f0e1a765d7fc38181, server=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,690 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 334949367d92cc18a393737b69358f37, server=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,690 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-26T21:49:56,691 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=61 2024-11-26T21:49:56,691 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=828b3f090c91,40359,1732657789839, table=hbase:meta, region=1588230740 2024-11-26T21:49:56,691 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 828b3f090c91,40359,1732657789839, state=OPEN 2024-11-26T21:49:56,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,692 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,692 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,692 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,693 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN to state=OFFLINE, location=null, table=test, region=c4145a9c4106377623484a3eb786fe8a to restore RIT 2024-11-26T21:49:56,693 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN to state=OFFLINE, location=null, table=test, region=334949367d92cc18a393737b69358f37 to restore RIT 2024-11-26T21:49:56,693 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN to state=OFFLINE, location=null, table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d to restore RIT 2024-11-26T21:49:56,693 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN to state=OFFLINE, location=null, table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181 to restore RIT 2024-11-26T21:49:56,693 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN to state=OFFLINE, location=null, table=test, region=21af909e3b8cf553daecee4e59c0898b to restore RIT 2024-11-26T21:49:56,693 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN to state=OFFLINE, location=null, table=test, region=bf200defbfcf83538e813f9fa91d9c35 to restore RIT 2024-11-26T21:49:56,693 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN to state=OFFLINE, location=null, table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d to restore RIT 2024-11-26T21:49:56,693 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN to state=OFFLINE, location=null, table=test, region=651cf53d9624e6cc98e46d6578c71496 to restore RIT 2024-11-26T21:49:56,694 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732657826694 2024-11-26T21:49:56,694 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T21:49:56,694 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,696 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T21:49:56,697 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T21:49:56,697 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e7efa0e8c09fd1f0e1a765d7fc38181, server=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,697 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:56,697 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e7efa0e8c09fd1f0e1a765d7fc38181, server=828b3f090c91,42273,1732657789877 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,701 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf200defbfcf83538e813f9fa91d9c35, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,701 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36699c847bc0e6c6b06fd8533d3c9edb, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,701 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34e003c01eb3f0a2e10e33152402164c, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,701 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a7c374815f888db179beadaf2489ad7, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=651cf53d9624e6cc98e46d6578c71496, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0a1ab633a90e6b5f5e9ba8ce6f66010, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8be22ac0f4cc7a17874b9d10ba204518, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a2d6b8d1f56582e3f52e3cfe7a116c1d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d22f8ae31d7e7a4d0c284e3b9676a393, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdd2d12bbd55678c7f638b4ef0fde3da, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=334949367d92cc18a393737b69358f37, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac402503c294574c0e555c45a12f0a4e, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06a1142ff2ac98bd39e6f73547d6aa98, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4145a9c4106377623484a3eb786fe8a, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a6bb8489f79432157c419605ac84965, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c5d2bc20a7b33de1c8b5a53f7fee57, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=996589dbb2419ea602bc5e348cbd05df, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cf2d9c6c6cf36bd61bb35f51df8f42, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b8b8290cc9d27b8d226e5dc2b7d770d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,704 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ef98738dc2598149d89e210ccdb4e6f, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,704 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=317db83a078d27fdfa108a4454669013, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,704 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21af909e3b8cf553daecee4e59c0898b, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,704 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f45cc001ffd649bcfbfe0f7af1630961, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,704 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e9187292024abc7067d470edad3ae67, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,704 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e7efa0e8c09fd1f0e1a765d7fc38181, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,704 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1062179709b27df36edcb7f6b054a4a6, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,704 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T21:49:56,704 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732657856704 2024-11-26T21:49:56,705 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732657916704 2024-11-26T21:49:56,705 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-26T21:49:56,705 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T21:49:56,705 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:56,705 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T21:49:56,705 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T21:49:56,705 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T21:49:56,705 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T21:49:56,705 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T21:49:56,705 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T21:49:56,706 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T21:49:56,710 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T21:49:56,710 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,710 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,710 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,710 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,710 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,710 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,710 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,710 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,710 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,710 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,710 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,710 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T21:49:56,710 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T21:49:56,710 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,710 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,710 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T21:49:56,710 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T21:49:56,710 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T21:49:56,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T21:49:56,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true 2024-11-26T21:49:56,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T21:49:56,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN 2024-11-26T21:49:56,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN 2024-11-26T21:49:56,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN 2024-11-26T21:49:56,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN 2024-11-26T21:49:56,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN 2024-11-26T21:49:56,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN 2024-11-26T21:49:56,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN 2024-11-26T21:49:56,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN 2024-11-26T21:49:56,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21af909e3b8cf553daecee4e59c0898b, server=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e7efa0e8c09fd1f0e1a765d7fc38181, server=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,713 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,713 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,714 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,714 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN 2024-11-26T21:49:56,714 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,714 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN 2024-11-26T21:49:56,714 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,714 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN 2024-11-26T21:49:56,714 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,714 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN 2024-11-26T21:49:56,714 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,714 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN 2024-11-26T21:49:56,714 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,714 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN 2024-11-26T21:49:56,714 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,714 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN 2024-11-26T21:49:56,714 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,714 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN 2024-11-26T21:49:56,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21af909e3b8cf553daecee4e59c0898b, server=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e7efa0e8c09fd1f0e1a765d7fc38181, server=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,714 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-26T21:49:56,715 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=61 2024-11-26T21:49:56,715 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=828b3f090c91,40359,1732657789839, table=hbase:meta, region=1588230740 2024-11-26T21:49:56,715 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 828b3f090c91,40359,1732657789839, state=OPEN 2024-11-26T21:49:56,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,717 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,717 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,717 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,718 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN to state=OFFLINE, location=null, table=test, region=c4145a9c4106377623484a3eb786fe8a to restore RIT 2024-11-26T21:49:56,718 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN to state=OFFLINE, location=null, table=test, region=334949367d92cc18a393737b69358f37 to restore RIT 2024-11-26T21:49:56,718 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN to state=OFFLINE, location=null, table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d to restore RIT 2024-11-26T21:49:56,718 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN to state=OFFLINE, location=null, table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181 to restore RIT 2024-11-26T21:49:56,718 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN to state=OFFLINE, location=null, table=test, region=21af909e3b8cf553daecee4e59c0898b to restore RIT 2024-11-26T21:49:56,718 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN to state=OFFLINE, location=null, table=test, region=bf200defbfcf83538e813f9fa91d9c35 to restore RIT 2024-11-26T21:49:56,718 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN to state=OFFLINE, location=null, table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d to restore RIT 2024-11-26T21:49:56,718 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN to state=OFFLINE, location=null, table=test, region=651cf53d9624e6cc98e46d6578c71496 to restore RIT 2024-11-26T21:49:56,719 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732657826719 2024-11-26T21:49:56,719 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T21:49:56,719 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,720 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T21:49:56,721 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T21:49:56,721 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21af909e3b8cf553daecee4e59c0898b, server=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,721 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:56,721 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21af909e3b8cf553daecee4e59c0898b, server=828b3f090c91,40359,1732657789839 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,726 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf200defbfcf83538e813f9fa91d9c35, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,727 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36699c847bc0e6c6b06fd8533d3c9edb, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,727 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34e003c01eb3f0a2e10e33152402164c, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,727 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a7c374815f888db179beadaf2489ad7, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,727 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=651cf53d9624e6cc98e46d6578c71496, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,728 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0a1ab633a90e6b5f5e9ba8ce6f66010, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,728 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8be22ac0f4cc7a17874b9d10ba204518, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,728 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a2d6b8d1f56582e3f52e3cfe7a116c1d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,728 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d22f8ae31d7e7a4d0c284e3b9676a393, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,728 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdd2d12bbd55678c7f638b4ef0fde3da, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=334949367d92cc18a393737b69358f37, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac402503c294574c0e555c45a12f0a4e, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06a1142ff2ac98bd39e6f73547d6aa98, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4145a9c4106377623484a3eb786fe8a, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a6bb8489f79432157c419605ac84965, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c5d2bc20a7b33de1c8b5a53f7fee57, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=996589dbb2419ea602bc5e348cbd05df, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cf2d9c6c6cf36bd61bb35f51df8f42, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b8b8290cc9d27b8d226e5dc2b7d770d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ef98738dc2598149d89e210ccdb4e6f, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=317db83a078d27fdfa108a4454669013, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21af909e3b8cf553daecee4e59c0898b, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f45cc001ffd649bcfbfe0f7af1630961, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e9187292024abc7067d470edad3ae67, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e7efa0e8c09fd1f0e1a765d7fc38181, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,732 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1062179709b27df36edcb7f6b054a4a6, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,732 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T21:49:56,732 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732657856732 2024-11-26T21:49:56,732 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732657916732 2024-11-26T21:49:56,732 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 13 msec 2024-11-26T21:49:56,732 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T21:49:56,732 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:56,732 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T21:49:56,732 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T21:49:56,733 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T21:49:56,733 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T21:49:56,733 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T21:49:56,733 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T21:49:56,734 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T21:49:56,738 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T21:49:56,738 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,738 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,738 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,738 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,738 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,738 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,738 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,738 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,738 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,738 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,738 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T21:49:56,738 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T21:49:56,738 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,738 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,738 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T21:49:56,738 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T21:49:56,739 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T21:49:56,740 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T21:49:56,742 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true 2024-11-26T21:49:56,742 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,742 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,742 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T21:49:56,742 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,742 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,742 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,742 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN 2024-11-26T21:49:56,743 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN 2024-11-26T21:49:56,743 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN 2024-11-26T21:49:56,743 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN 2024-11-26T21:49:56,743 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN 2024-11-26T21:49:56,743 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN 2024-11-26T21:49:56,743 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN 2024-11-26T21:49:56,743 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN 2024-11-26T21:49:56,743 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,743 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21af909e3b8cf553daecee4e59c0898b, server=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,743 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,743 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,743 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,743 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,743 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,743 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,743 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,743 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,743 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN 2024-11-26T21:49:56,743 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,743 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN 2024-11-26T21:49:56,744 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,744 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN 2024-11-26T21:49:56,744 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,744 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN 2024-11-26T21:49:56,744 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,744 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN 2024-11-26T21:49:56,744 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,744 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN 2024-11-26T21:49:56,744 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,744 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN 2024-11-26T21:49:56,744 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,744 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN 2024-11-26T21:49:56,744 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,744 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21af909e3b8cf553daecee4e59c0898b, server=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,744 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-26T21:49:56,745 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=61 2024-11-26T21:49:56,745 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=828b3f090c91,40359,1732657789839, table=hbase:meta, region=1588230740 2024-11-26T21:49:56,746 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 828b3f090c91,40359,1732657789839, state=OPEN 2024-11-26T21:49:56,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,747 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,747 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,747 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,748 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN to state=OFFLINE, location=null, table=test, region=c4145a9c4106377623484a3eb786fe8a to restore RIT 2024-11-26T21:49:56,748 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN to state=OFFLINE, location=null, table=test, region=334949367d92cc18a393737b69358f37 to restore RIT 2024-11-26T21:49:56,748 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN to state=OFFLINE, location=null, table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d to restore RIT 2024-11-26T21:49:56,748 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN to state=OFFLINE, location=null, table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181 to restore RIT 2024-11-26T21:49:56,748 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN to state=OFFLINE, location=null, table=test, region=21af909e3b8cf553daecee4e59c0898b to restore RIT 2024-11-26T21:49:56,748 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN to state=OFFLINE, location=null, table=test, region=bf200defbfcf83538e813f9fa91d9c35 to restore RIT 2024-11-26T21:49:56,748 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN to state=OFFLINE, location=null, table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d to restore RIT 2024-11-26T21:49:56,748 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN to state=OFFLINE, location=null, table=test, region=651cf53d9624e6cc98e46d6578c71496 to restore RIT 2024-11-26T21:49:56,749 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732657826749 2024-11-26T21:49:56,749 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T21:49:56,749 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,750 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T21:49:56,751 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T21:49:56,751 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,752 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:56,752 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,755 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf200defbfcf83538e813f9fa91d9c35, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,755 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36699c847bc0e6c6b06fd8533d3c9edb, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,755 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34e003c01eb3f0a2e10e33152402164c, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,755 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a7c374815f888db179beadaf2489ad7, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,755 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=651cf53d9624e6cc98e46d6578c71496, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,755 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0a1ab633a90e6b5f5e9ba8ce6f66010, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,756 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8be22ac0f4cc7a17874b9d10ba204518, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,756 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a2d6b8d1f56582e3f52e3cfe7a116c1d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,756 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d22f8ae31d7e7a4d0c284e3b9676a393, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,756 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdd2d12bbd55678c7f638b4ef0fde3da, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,756 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=334949367d92cc18a393737b69358f37, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,756 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac402503c294574c0e555c45a12f0a4e, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,756 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06a1142ff2ac98bd39e6f73547d6aa98, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,756 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4145a9c4106377623484a3eb786fe8a, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,756 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a6bb8489f79432157c419605ac84965, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,757 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c5d2bc20a7b33de1c8b5a53f7fee57, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,757 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=996589dbb2419ea602bc5e348cbd05df, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,757 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cf2d9c6c6cf36bd61bb35f51df8f42, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,757 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b8b8290cc9d27b8d226e5dc2b7d770d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,757 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ef98738dc2598149d89e210ccdb4e6f, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,757 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=317db83a078d27fdfa108a4454669013, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,757 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21af909e3b8cf553daecee4e59c0898b, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,757 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f45cc001ffd649bcfbfe0f7af1630961, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,758 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e9187292024abc7067d470edad3ae67, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,758 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e7efa0e8c09fd1f0e1a765d7fc38181, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,758 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1062179709b27df36edcb7f6b054a4a6, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,758 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T21:49:56,758 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732657856758 2024-11-26T21:49:56,758 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732657916758 2024-11-26T21:49:56,758 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-26T21:49:56,758 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T21:49:56,758 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:56,758 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T21:49:56,758 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T21:49:56,759 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T21:49:56,759 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T21:49:56,759 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T21:49:56,759 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T21:49:56,759 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T21:49:56,762 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T21:49:56,762 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,762 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,762 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,762 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,762 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,762 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,762 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,762 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,762 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,762 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T21:49:56,762 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T21:49:56,762 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,762 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,762 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T21:49:56,762 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T21:49:56,763 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T21:49:56,763 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T21:49:56,765 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true 2024-11-26T21:49:56,765 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,765 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,765 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T21:49:56,765 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,765 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,765 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,766 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN 2024-11-26T21:49:56,766 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN 2024-11-26T21:49:56,766 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN 2024-11-26T21:49:56,766 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN 2024-11-26T21:49:56,766 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN 2024-11-26T21:49:56,766 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN 2024-11-26T21:49:56,766 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN 2024-11-26T21:49:56,766 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN 2024-11-26T21:49:56,766 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,766 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,766 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,766 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,766 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,766 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,766 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,766 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,766 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,766 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN 2024-11-26T21:49:56,766 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,766 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN 2024-11-26T21:49:56,766 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,766 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN 2024-11-26T21:49:56,766 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,766 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN 2024-11-26T21:49:56,766 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,766 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN 2024-11-26T21:49:56,766 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,766 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN 2024-11-26T21:49:56,766 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,767 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN 2024-11-26T21:49:56,767 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,767 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN 2024-11-26T21:49:56,767 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,767 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-26T21:49:56,767 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=61 2024-11-26T21:49:56,768 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=828b3f090c91,40359,1732657789839, table=hbase:meta, region=1588230740 2024-11-26T21:49:56,768 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 828b3f090c91,40359,1732657789839, state=OPEN 2024-11-26T21:49:56,769 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,769 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,769 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,769 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,769 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,769 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,770 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN to state=OFFLINE, location=null, table=test, region=c4145a9c4106377623484a3eb786fe8a to restore RIT 2024-11-26T21:49:56,770 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN to state=OFFLINE, location=null, table=test, region=334949367d92cc18a393737b69358f37 to restore RIT 2024-11-26T21:49:56,770 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN to state=OFFLINE, location=null, table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d to restore RIT 2024-11-26T21:49:56,770 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN to state=OFFLINE, location=null, table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181 to restore RIT 2024-11-26T21:49:56,770 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN to state=OFFLINE, location=null, table=test, region=21af909e3b8cf553daecee4e59c0898b to restore RIT 2024-11-26T21:49:56,770 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN to state=OFFLINE, location=null, table=test, region=bf200defbfcf83538e813f9fa91d9c35 to restore RIT 2024-11-26T21:49:56,770 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN to state=OFFLINE, location=null, table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d to restore RIT 2024-11-26T21:49:56,770 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN to state=OFFLINE, location=null, table=test, region=651cf53d9624e6cc98e46d6578c71496 to restore RIT 2024-11-26T21:49:56,770 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732657826770 2024-11-26T21:49:56,770 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T21:49:56,770 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,772 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T21:49:56,773 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T21:49:56,773 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN 2024-11-26T21:49:56,773 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:56,773 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,777 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf200defbfcf83538e813f9fa91d9c35, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,777 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36699c847bc0e6c6b06fd8533d3c9edb, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,777 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34e003c01eb3f0a2e10e33152402164c, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,777 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a7c374815f888db179beadaf2489ad7, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,777 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=651cf53d9624e6cc98e46d6578c71496, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,778 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0a1ab633a90e6b5f5e9ba8ce6f66010, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,778 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8be22ac0f4cc7a17874b9d10ba204518, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,778 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a2d6b8d1f56582e3f52e3cfe7a116c1d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,778 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d22f8ae31d7e7a4d0c284e3b9676a393, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,778 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdd2d12bbd55678c7f638b4ef0fde3da, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,779 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=334949367d92cc18a393737b69358f37, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,779 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac402503c294574c0e555c45a12f0a4e, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,779 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06a1142ff2ac98bd39e6f73547d6aa98, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,779 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4145a9c4106377623484a3eb786fe8a, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,779 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a6bb8489f79432157c419605ac84965, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,779 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c5d2bc20a7b33de1c8b5a53f7fee57, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,780 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=996589dbb2419ea602bc5e348cbd05df, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,780 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cf2d9c6c6cf36bd61bb35f51df8f42, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,780 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b8b8290cc9d27b8d226e5dc2b7d770d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,780 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ef98738dc2598149d89e210ccdb4e6f, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,780 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=317db83a078d27fdfa108a4454669013, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,780 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21af909e3b8cf553daecee4e59c0898b, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,780 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f45cc001ffd649bcfbfe0f7af1630961, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,780 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e9187292024abc7067d470edad3ae67, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,780 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e7efa0e8c09fd1f0e1a765d7fc38181, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,781 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1062179709b27df36edcb7f6b054a4a6, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,781 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T21:49:56,781 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732657856781 2024-11-26T21:49:56,781 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732657916781 2024-11-26T21:49:56,781 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-26T21:49:56,781 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T21:49:56,781 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:56,781 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T21:49:56,781 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T21:49:56,781 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T21:49:56,781 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T21:49:56,781 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T21:49:56,782 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T21:49:56,782 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T21:49:56,784 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T21:49:56,784 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,784 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,784 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,784 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,784 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,784 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,784 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,784 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,785 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T21:49:56,785 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T21:49:56,785 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,785 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,785 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T21:49:56,785 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T21:49:56,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T21:49:56,786 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T21:49:56,788 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true 2024-11-26T21:49:56,788 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,788 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,788 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T21:49:56,788 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,788 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44203 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,788 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,788 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,788 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN 2024-11-26T21:49:56,788 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN 2024-11-26T21:49:56,788 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN 2024-11-26T21:49:56,788 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN 2024-11-26T21:49:56,788 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN 2024-11-26T21:49:56,788 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN 2024-11-26T21:49:56,788 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN 2024-11-26T21:49:56,788 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN 2024-11-26T21:49:56,788 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,788 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,788 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,788 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,788 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,789 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,789 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,789 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,789 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN 2024-11-26T21:49:56,789 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,789 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN 2024-11-26T21:49:56,789 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,789 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN 2024-11-26T21:49:56,789 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,789 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN 2024-11-26T21:49:56,789 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,789 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN 2024-11-26T21:49:56,789 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,789 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN 2024-11-26T21:49:56,789 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,789 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN 2024-11-26T21:49:56,789 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,789 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN 2024-11-26T21:49:56,790 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-26T21:49:56,790 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44203 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,791 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=61 2024-11-26T21:49:56,791 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=828b3f090c91,40359,1732657789839, table=hbase:meta, region=1588230740 2024-11-26T21:49:56,791 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 828b3f090c91,40359,1732657789839, state=OPEN 2024-11-26T21:49:56,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,792 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,792 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,793 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,793 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN to state=OFFLINE, location=null, table=test, region=c4145a9c4106377623484a3eb786fe8a to restore RIT 2024-11-26T21:49:56,793 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN to state=OFFLINE, location=null, table=test, region=334949367d92cc18a393737b69358f37 to restore RIT 2024-11-26T21:49:56,793 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN to state=OFFLINE, location=null, table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d to restore RIT 2024-11-26T21:49:56,793 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN to state=OFFLINE, location=null, table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181 to restore RIT 2024-11-26T21:49:56,793 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN to state=OFFLINE, location=null, table=test, region=21af909e3b8cf553daecee4e59c0898b to restore RIT 2024-11-26T21:49:56,793 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN to state=OFFLINE, location=null, table=test, region=bf200defbfcf83538e813f9fa91d9c35 to restore RIT 2024-11-26T21:49:56,793 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN to state=OFFLINE, location=null, table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d to restore RIT 2024-11-26T21:49:56,794 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=651cf53d9624e6cc98e46d6578c71496, ASSIGN to state=OFFLINE, location=null, table=test, region=651cf53d9624e6cc98e46d6578c71496 to restore RIT 2024-11-26T21:49:56,798 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732657826798 2024-11-26T21:49:56,799 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T21:49:56,799 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,800 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T21:49:56,801 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T21:49:56,801 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN 2024-11-26T21:49:56,801 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:56,801 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf200defbfcf83538e813f9fa91d9c35, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36699c847bc0e6c6b06fd8533d3c9edb, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34e003c01eb3f0a2e10e33152402164c, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a7c374815f888db179beadaf2489ad7, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=651cf53d9624e6cc98e46d6578c71496, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0a1ab633a90e6b5f5e9ba8ce6f66010, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8be22ac0f4cc7a17874b9d10ba204518, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a2d6b8d1f56582e3f52e3cfe7a116c1d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d22f8ae31d7e7a4d0c284e3b9676a393, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdd2d12bbd55678c7f638b4ef0fde3da, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=334949367d92cc18a393737b69358f37, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac402503c294574c0e555c45a12f0a4e, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06a1142ff2ac98bd39e6f73547d6aa98, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4145a9c4106377623484a3eb786fe8a, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a6bb8489f79432157c419605ac84965, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c5d2bc20a7b33de1c8b5a53f7fee57, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=996589dbb2419ea602bc5e348cbd05df, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cf2d9c6c6cf36bd61bb35f51df8f42, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b8b8290cc9d27b8d226e5dc2b7d770d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ef98738dc2598149d89e210ccdb4e6f, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=317db83a078d27fdfa108a4454669013, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21af909e3b8cf553daecee4e59c0898b, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f45cc001ffd649bcfbfe0f7af1630961, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e9187292024abc7067d470edad3ae67, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e7efa0e8c09fd1f0e1a765d7fc38181, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1062179709b27df36edcb7f6b054a4a6, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,809 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T21:49:56,809 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732657856809 2024-11-26T21:49:56,809 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732657916809 2024-11-26T21:49:56,809 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-26T21:49:56,809 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T21:49:56,809 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:56,809 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T21:49:56,810 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T21:49:56,810 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T21:49:56,810 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T21:49:56,810 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T21:49:56,810 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T21:49:56,811 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T21:49:56,813 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T21:49:56,813 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,813 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,814 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,814 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,814 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,814 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,814 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,814 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T21:49:56,814 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T21:49:56,814 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,814 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,814 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T21:49:56,814 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T21:49:56,814 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T21:49:56,815 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T21:49:56,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true 2024-11-26T21:49:56,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T21:49:56,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN 2024-11-26T21:49:56,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN 2024-11-26T21:49:56,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN 2024-11-26T21:49:56,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN 2024-11-26T21:49:56,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN 2024-11-26T21:49:56,817 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN 2024-11-26T21:49:56,817 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN 2024-11-26T21:49:56,817 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,817 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,817 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,817 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,817 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,817 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,817 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,817 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,817 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN 2024-11-26T21:49:56,817 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,817 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN 2024-11-26T21:49:56,817 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,817 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN 2024-11-26T21:49:56,817 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,817 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN 2024-11-26T21:49:56,817 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,817 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN 2024-11-26T21:49:56,817 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,817 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN 2024-11-26T21:49:56,817 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,817 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN 2024-11-26T21:49:56,817 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-26T21:49:56,818 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=61 2024-11-26T21:49:56,818 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=828b3f090c91,40359,1732657789839, table=hbase:meta, region=1588230740 2024-11-26T21:49:56,818 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 828b3f090c91,40359,1732657789839, state=OPEN 2024-11-26T21:49:56,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,820 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,820 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,820 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,820 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN to state=OFFLINE, location=null, table=test, region=c4145a9c4106377623484a3eb786fe8a to restore RIT 2024-11-26T21:49:56,820 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN to state=OFFLINE, location=null, table=test, region=334949367d92cc18a393737b69358f37 to restore RIT 2024-11-26T21:49:56,821 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN to state=OFFLINE, location=null, table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d to restore RIT 2024-11-26T21:49:56,821 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN to state=OFFLINE, location=null, table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181 to restore RIT 2024-11-26T21:49:56,821 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN to state=OFFLINE, location=null, table=test, region=21af909e3b8cf553daecee4e59c0898b to restore RIT 2024-11-26T21:49:56,821 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN to state=OFFLINE, location=null, table=test, region=bf200defbfcf83538e813f9fa91d9c35 to restore RIT 2024-11-26T21:49:56,821 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d, ASSIGN to state=OFFLINE, location=null, table=test, region=2b8b8290cc9d27b8d226e5dc2b7d770d to restore RIT 2024-11-26T21:49:56,821 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732657826821 2024-11-26T21:49:56,821 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T21:49:56,821 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,823 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T21:49:56,825 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T21:49:56,825 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN 2024-11-26T21:49:56,825 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:56,825 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,828 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf200defbfcf83538e813f9fa91d9c35, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,828 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36699c847bc0e6c6b06fd8533d3c9edb, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,828 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34e003c01eb3f0a2e10e33152402164c, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,828 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a7c374815f888db179beadaf2489ad7, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,828 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=651cf53d9624e6cc98e46d6578c71496, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,829 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0a1ab633a90e6b5f5e9ba8ce6f66010, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,829 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8be22ac0f4cc7a17874b9d10ba204518, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,829 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a2d6b8d1f56582e3f52e3cfe7a116c1d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,829 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d22f8ae31d7e7a4d0c284e3b9676a393, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,829 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdd2d12bbd55678c7f638b4ef0fde3da, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,829 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=334949367d92cc18a393737b69358f37, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,830 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac402503c294574c0e555c45a12f0a4e, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,830 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06a1142ff2ac98bd39e6f73547d6aa98, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,830 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4145a9c4106377623484a3eb786fe8a, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,830 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a6bb8489f79432157c419605ac84965, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,830 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c5d2bc20a7b33de1c8b5a53f7fee57, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,830 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=996589dbb2419ea602bc5e348cbd05df, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,831 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cf2d9c6c6cf36bd61bb35f51df8f42, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,831 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b8b8290cc9d27b8d226e5dc2b7d770d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,831 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ef98738dc2598149d89e210ccdb4e6f, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,831 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=317db83a078d27fdfa108a4454669013, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,831 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21af909e3b8cf553daecee4e59c0898b, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,831 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f45cc001ffd649bcfbfe0f7af1630961, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,832 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e9187292024abc7067d470edad3ae67, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,832 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e7efa0e8c09fd1f0e1a765d7fc38181, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,832 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1062179709b27df36edcb7f6b054a4a6, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,832 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T21:49:56,832 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732657856832 2024-11-26T21:49:56,832 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732657916832 2024-11-26T21:49:56,832 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-26T21:49:56,832 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T21:49:56,832 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:56,832 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T21:49:56,832 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T21:49:56,833 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T21:49:56,833 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T21:49:56,833 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T21:49:56,833 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T21:49:56,834 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T21:49:56,836 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T21:49:56,836 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,836 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,837 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,837 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,837 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,837 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,837 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T21:49:56,837 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T21:49:56,837 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,837 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,837 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T21:49:56,837 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T21:49:56,837 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T21:49:56,838 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T21:49:56,839 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true 2024-11-26T21:49:56,839 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,839 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,839 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T21:49:56,840 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,840 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,840 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,840 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN 2024-11-26T21:49:56,840 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN 2024-11-26T21:49:56,840 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN 2024-11-26T21:49:56,840 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN 2024-11-26T21:49:56,840 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN 2024-11-26T21:49:56,840 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN 2024-11-26T21:49:56,840 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,840 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,840 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,840 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,840 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,840 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,840 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,840 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,840 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN 2024-11-26T21:49:56,840 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,840 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN 2024-11-26T21:49:56,841 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,841 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN 2024-11-26T21:49:56,841 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,841 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN 2024-11-26T21:49:56,841 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,841 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN 2024-11-26T21:49:56,841 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,841 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN 2024-11-26T21:49:56,841 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-26T21:49:56,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=61 2024-11-26T21:49:56,842 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=828b3f090c91,40359,1732657789839, table=hbase:meta, region=1588230740 2024-11-26T21:49:56,842 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 828b3f090c91,40359,1732657789839, state=OPEN 2024-11-26T21:49:56,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,844 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,844 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,844 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,845 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN to state=OFFLINE, location=null, table=test, region=c4145a9c4106377623484a3eb786fe8a to restore RIT 2024-11-26T21:49:56,845 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN to state=OFFLINE, location=null, table=test, region=334949367d92cc18a393737b69358f37 to restore RIT 2024-11-26T21:49:56,845 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN to state=OFFLINE, location=null, table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d to restore RIT 2024-11-26T21:49:56,845 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN to state=OFFLINE, location=null, table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181 to restore RIT 2024-11-26T21:49:56,845 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN to state=OFFLINE, location=null, table=test, region=21af909e3b8cf553daecee4e59c0898b to restore RIT 2024-11-26T21:49:56,845 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf200defbfcf83538e813f9fa91d9c35, ASSIGN to state=OFFLINE, location=null, table=test, region=bf200defbfcf83538e813f9fa91d9c35 to restore RIT 2024-11-26T21:49:56,849 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732657826849 2024-11-26T21:49:56,849 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T21:49:56,849 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,850 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T21:49:56,852 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T21:49:56,852 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN 2024-11-26T21:49:56,852 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:56,852 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,855 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf200defbfcf83538e813f9fa91d9c35, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,855 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36699c847bc0e6c6b06fd8533d3c9edb, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,856 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34e003c01eb3f0a2e10e33152402164c, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,856 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a7c374815f888db179beadaf2489ad7, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,856 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=651cf53d9624e6cc98e46d6578c71496, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,856 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0a1ab633a90e6b5f5e9ba8ce6f66010, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,856 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8be22ac0f4cc7a17874b9d10ba204518, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,856 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a2d6b8d1f56582e3f52e3cfe7a116c1d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,856 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d22f8ae31d7e7a4d0c284e3b9676a393, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,857 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdd2d12bbd55678c7f638b4ef0fde3da, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,857 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=334949367d92cc18a393737b69358f37, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,857 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac402503c294574c0e555c45a12f0a4e, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,857 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06a1142ff2ac98bd39e6f73547d6aa98, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,857 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4145a9c4106377623484a3eb786fe8a, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,857 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a6bb8489f79432157c419605ac84965, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,857 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c5d2bc20a7b33de1c8b5a53f7fee57, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,857 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=996589dbb2419ea602bc5e348cbd05df, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,858 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cf2d9c6c6cf36bd61bb35f51df8f42, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,858 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b8b8290cc9d27b8d226e5dc2b7d770d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,858 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ef98738dc2598149d89e210ccdb4e6f, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,858 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=317db83a078d27fdfa108a4454669013, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,858 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21af909e3b8cf553daecee4e59c0898b, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,858 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f45cc001ffd649bcfbfe0f7af1630961, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,858 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e9187292024abc7067d470edad3ae67, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,859 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e7efa0e8c09fd1f0e1a765d7fc38181, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,859 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1062179709b27df36edcb7f6b054a4a6, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,859 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T21:49:56,859 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732657856859 2024-11-26T21:49:56,859 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732657916859 2024-11-26T21:49:56,859 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-26T21:49:56,859 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T21:49:56,859 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:56,859 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T21:49:56,859 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T21:49:56,860 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T21:49:56,860 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T21:49:56,860 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T21:49:56,860 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T21:49:56,861 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T21:49:56,863 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T21:49:56,863 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,863 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,863 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,863 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,863 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,863 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T21:49:56,863 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T21:49:56,863 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,864 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,864 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T21:49:56,864 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T21:49:56,864 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T21:49:56,865 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T21:49:56,866 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true 2024-11-26T21:49:56,866 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,866 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,866 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T21:49:56,866 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,866 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,866 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,866 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN 2024-11-26T21:49:56,866 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN 2024-11-26T21:49:56,866 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN 2024-11-26T21:49:56,866 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN 2024-11-26T21:49:56,866 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN 2024-11-26T21:49:56,867 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,867 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,867 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,867 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,867 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,867 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,867 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,867 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,867 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN 2024-11-26T21:49:56,867 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,867 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN 2024-11-26T21:49:56,867 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,867 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN 2024-11-26T21:49:56,867 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,867 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN 2024-11-26T21:49:56,867 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,867 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN 2024-11-26T21:49:56,867 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-26T21:49:56,868 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=61 2024-11-26T21:49:56,869 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=828b3f090c91,40359,1732657789839, table=hbase:meta, region=1588230740 2024-11-26T21:49:56,869 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 828b3f090c91,40359,1732657789839, state=OPEN 2024-11-26T21:49:56,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,870 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,870 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,870 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,871 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN to state=OFFLINE, location=null, table=test, region=c4145a9c4106377623484a3eb786fe8a to restore RIT 2024-11-26T21:49:56,871 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN to state=OFFLINE, location=null, table=test, region=334949367d92cc18a393737b69358f37 to restore RIT 2024-11-26T21:49:56,871 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN to state=OFFLINE, location=null, table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d to restore RIT 2024-11-26T21:49:56,871 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN to state=OFFLINE, location=null, table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181 to restore RIT 2024-11-26T21:49:56,872 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=21af909e3b8cf553daecee4e59c0898b, ASSIGN to state=OFFLINE, location=null, table=test, region=21af909e3b8cf553daecee4e59c0898b to restore RIT 2024-11-26T21:49:56,872 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732657826872 2024-11-26T21:49:56,872 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T21:49:56,872 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,874 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T21:49:56,877 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T21:49:56,878 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN 2024-11-26T21:49:56,878 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:56,878 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf200defbfcf83538e813f9fa91d9c35, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36699c847bc0e6c6b06fd8533d3c9edb, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34e003c01eb3f0a2e10e33152402164c, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a7c374815f888db179beadaf2489ad7, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,879 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=651cf53d9624e6cc98e46d6578c71496, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,879 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0a1ab633a90e6b5f5e9ba8ce6f66010, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,879 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8be22ac0f4cc7a17874b9d10ba204518, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,879 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a2d6b8d1f56582e3f52e3cfe7a116c1d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,879 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d22f8ae31d7e7a4d0c284e3b9676a393, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,879 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdd2d12bbd55678c7f638b4ef0fde3da, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,879 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=334949367d92cc18a393737b69358f37, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,880 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac402503c294574c0e555c45a12f0a4e, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,880 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06a1142ff2ac98bd39e6f73547d6aa98, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,880 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4145a9c4106377623484a3eb786fe8a, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,880 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a6bb8489f79432157c419605ac84965, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,880 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c5d2bc20a7b33de1c8b5a53f7fee57, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,880 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=996589dbb2419ea602bc5e348cbd05df, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,880 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cf2d9c6c6cf36bd61bb35f51df8f42, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,880 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b8b8290cc9d27b8d226e5dc2b7d770d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,881 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ef98738dc2598149d89e210ccdb4e6f, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,881 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=317db83a078d27fdfa108a4454669013, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,881 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21af909e3b8cf553daecee4e59c0898b, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,881 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f45cc001ffd649bcfbfe0f7af1630961, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,881 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e9187292024abc7067d470edad3ae67, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,881 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e7efa0e8c09fd1f0e1a765d7fc38181, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,881 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1062179709b27df36edcb7f6b054a4a6, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,881 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T21:49:56,882 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732657856881 2024-11-26T21:49:56,882 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732657916882 2024-11-26T21:49:56,882 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-26T21:49:56,882 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T21:49:56,882 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:56,882 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T21:49:56,882 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T21:49:56,882 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T21:49:56,882 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T21:49:56,882 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T21:49:56,883 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T21:49:56,883 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T21:49:56,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T21:49:56,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T21:49:56,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T21:49:56,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,887 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T21:49:56,887 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T21:49:56,887 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T21:49:56,887 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T21:49:56,889 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true 2024-11-26T21:49:56,889 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,889 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,889 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T21:49:56,889 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,889 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,889 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,889 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN 2024-11-26T21:49:56,889 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN 2024-11-26T21:49:56,889 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN 2024-11-26T21:49:56,889 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN 2024-11-26T21:49:56,889 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,889 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,889 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,889 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,889 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,889 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,889 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,890 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,890 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN 2024-11-26T21:49:56,890 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,890 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN 2024-11-26T21:49:56,890 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,890 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN 2024-11-26T21:49:56,890 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,890 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN 2024-11-26T21:49:56,890 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-26T21:49:56,891 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=61 2024-11-26T21:49:56,891 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=828b3f090c91,40359,1732657789839, table=hbase:meta, region=1588230740 2024-11-26T21:49:56,891 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44203 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,891 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 828b3f090c91,40359,1732657789839, state=OPEN 2024-11-26T21:49:56,893 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,893 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,893 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,893 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,893 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,893 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,894 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44203 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,895 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN to state=OFFLINE, location=null, table=test, region=c4145a9c4106377623484a3eb786fe8a to restore RIT 2024-11-26T21:49:56,895 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN to state=OFFLINE, location=null, table=test, region=334949367d92cc18a393737b69358f37 to restore RIT 2024-11-26T21:49:56,895 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN to state=OFFLINE, location=null, table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d to restore RIT 2024-11-26T21:49:56,895 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181, ASSIGN to state=OFFLINE, location=null, table=test, region=1e7efa0e8c09fd1f0e1a765d7fc38181 to restore RIT 2024-11-26T21:49:56,906 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732657826906 2024-11-26T21:49:56,906 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T21:49:56,906 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,907 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T21:49:56,908 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T21:49:56,909 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN 2024-11-26T21:49:56,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:56,909 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,913 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf200defbfcf83538e813f9fa91d9c35, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,913 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36699c847bc0e6c6b06fd8533d3c9edb, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,913 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34e003c01eb3f0a2e10e33152402164c, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,913 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a7c374815f888db179beadaf2489ad7, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,913 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=651cf53d9624e6cc98e46d6578c71496, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,913 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0a1ab633a90e6b5f5e9ba8ce6f66010, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,913 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8be22ac0f4cc7a17874b9d10ba204518, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,914 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a2d6b8d1f56582e3f52e3cfe7a116c1d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,914 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d22f8ae31d7e7a4d0c284e3b9676a393, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,914 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdd2d12bbd55678c7f638b4ef0fde3da, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,914 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=334949367d92cc18a393737b69358f37, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,914 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac402503c294574c0e555c45a12f0a4e, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,914 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06a1142ff2ac98bd39e6f73547d6aa98, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,915 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4145a9c4106377623484a3eb786fe8a, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,915 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a6bb8489f79432157c419605ac84965, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,915 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c5d2bc20a7b33de1c8b5a53f7fee57, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,915 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=996589dbb2419ea602bc5e348cbd05df, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,915 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cf2d9c6c6cf36bd61bb35f51df8f42, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,915 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b8b8290cc9d27b8d226e5dc2b7d770d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,915 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ef98738dc2598149d89e210ccdb4e6f, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,915 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=317db83a078d27fdfa108a4454669013, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,916 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21af909e3b8cf553daecee4e59c0898b, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,916 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f45cc001ffd649bcfbfe0f7af1630961, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,916 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e9187292024abc7067d470edad3ae67, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,916 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e7efa0e8c09fd1f0e1a765d7fc38181, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,916 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1062179709b27df36edcb7f6b054a4a6, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,916 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T21:49:56,916 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732657856916 2024-11-26T21:49:56,916 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732657916916 2024-11-26T21:49:56,916 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-26T21:49:56,916 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T21:49:56,916 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:56,917 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T21:49:56,917 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T21:49:56,917 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T21:49:56,917 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T21:49:56,917 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T21:49:56,917 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T21:49:56,918 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T21:49:56,920 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T21:49:56,920 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,920 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,920 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,920 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T21:49:56,920 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T21:49:56,920 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,920 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,920 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T21:49:56,921 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T21:49:56,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T21:49:56,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T21:49:56,922 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true 2024-11-26T21:49:56,923 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,923 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,923 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T21:49:56,923 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,923 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,923 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,923 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN 2024-11-26T21:49:56,923 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN 2024-11-26T21:49:56,923 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN 2024-11-26T21:49:56,923 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,923 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,923 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,923 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,923 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,923 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,923 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,923 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,924 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN 2024-11-26T21:49:56,924 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,924 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN 2024-11-26T21:49:56,924 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,924 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN 2024-11-26T21:49:56,924 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-26T21:49:56,925 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=61 2024-11-26T21:49:56,925 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=828b3f090c91,40359,1732657789839, table=hbase:meta, region=1588230740 2024-11-26T21:49:56,925 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 828b3f090c91,40359,1732657789839, state=OPEN 2024-11-26T21:49:56,926 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,926 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,927 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,927 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,927 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,927 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN to state=OFFLINE, location=null, table=test, region=c4145a9c4106377623484a3eb786fe8a to restore RIT 2024-11-26T21:49:56,927 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN to state=OFFLINE, location=null, table=test, region=334949367d92cc18a393737b69358f37 to restore RIT 2024-11-26T21:49:56,927 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d, ASSIGN to state=OFFLINE, location=null, table=test, region=a2d6b8d1f56582e3f52e3cfe7a116c1d to restore RIT 2024-11-26T21:49:56,928 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732657826928 2024-11-26T21:49:56,928 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T21:49:56,928 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,929 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T21:49:56,930 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T21:49:56,930 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN 2024-11-26T21:49:56,930 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:56,930 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,933 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf200defbfcf83538e813f9fa91d9c35, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,933 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36699c847bc0e6c6b06fd8533d3c9edb, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,933 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34e003c01eb3f0a2e10e33152402164c, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,934 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a7c374815f888db179beadaf2489ad7, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,934 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=651cf53d9624e6cc98e46d6578c71496, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,934 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0a1ab633a90e6b5f5e9ba8ce6f66010, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,934 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8be22ac0f4cc7a17874b9d10ba204518, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,934 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a2d6b8d1f56582e3f52e3cfe7a116c1d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,934 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d22f8ae31d7e7a4d0c284e3b9676a393, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,934 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdd2d12bbd55678c7f638b4ef0fde3da, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,934 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=334949367d92cc18a393737b69358f37, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac402503c294574c0e555c45a12f0a4e, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06a1142ff2ac98bd39e6f73547d6aa98, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4145a9c4106377623484a3eb786fe8a, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a6bb8489f79432157c419605ac84965, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c5d2bc20a7b33de1c8b5a53f7fee57, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=996589dbb2419ea602bc5e348cbd05df, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cf2d9c6c6cf36bd61bb35f51df8f42, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b8b8290cc9d27b8d226e5dc2b7d770d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ef98738dc2598149d89e210ccdb4e6f, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=317db83a078d27fdfa108a4454669013, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21af909e3b8cf553daecee4e59c0898b, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f45cc001ffd649bcfbfe0f7af1630961, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e9187292024abc7067d470edad3ae67, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e7efa0e8c09fd1f0e1a765d7fc38181, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1062179709b27df36edcb7f6b054a4a6, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,936 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T21:49:56,936 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732657856936 2024-11-26T21:49:56,936 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732657916936 2024-11-26T21:49:56,937 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-26T21:49:56,937 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T21:49:56,937 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:56,937 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T21:49:56,937 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T21:49:56,937 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T21:49:56,937 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T21:49:56,937 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T21:49:56,937 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T21:49:56,938 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T21:49:56,940 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T21:49:56,940 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,940 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,940 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T21:49:56,940 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T21:49:56,940 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,940 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,941 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T21:49:56,941 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T21:49:56,941 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T21:49:56,941 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T21:49:56,943 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true 2024-11-26T21:49:56,943 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,943 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,943 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T21:49:56,943 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,943 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,943 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,943 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN 2024-11-26T21:49:56,943 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN 2024-11-26T21:49:56,943 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,943 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,943 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,943 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,943 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,943 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,943 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,943 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,944 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN 2024-11-26T21:49:56,944 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,944 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN 2024-11-26T21:49:56,944 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-26T21:49:56,945 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=61 2024-11-26T21:49:56,945 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=828b3f090c91,40359,1732657789839, table=hbase:meta, region=1588230740 2024-11-26T21:49:56,945 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 828b3f090c91,40359,1732657789839, state=OPEN 2024-11-26T21:49:56,947 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,947 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,947 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,947 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,947 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,948 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,949 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN to state=OFFLINE, location=null, table=test, region=c4145a9c4106377623484a3eb786fe8a to restore RIT 2024-11-26T21:49:56,949 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=334949367d92cc18a393737b69358f37, ASSIGN to state=OFFLINE, location=null, table=test, region=334949367d92cc18a393737b69358f37 to restore RIT 2024-11-26T21:49:56,953 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732657826953 2024-11-26T21:49:56,953 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T21:49:56,953 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,955 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T21:49:56,956 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T21:49:56,956 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN 2024-11-26T21:49:56,956 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:56,956 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,959 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf200defbfcf83538e813f9fa91d9c35, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,959 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36699c847bc0e6c6b06fd8533d3c9edb, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,959 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34e003c01eb3f0a2e10e33152402164c, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,959 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a7c374815f888db179beadaf2489ad7, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,959 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=651cf53d9624e6cc98e46d6578c71496, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,959 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0a1ab633a90e6b5f5e9ba8ce6f66010, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,959 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8be22ac0f4cc7a17874b9d10ba204518, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,959 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a2d6b8d1f56582e3f52e3cfe7a116c1d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,959 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d22f8ae31d7e7a4d0c284e3b9676a393, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,960 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdd2d12bbd55678c7f638b4ef0fde3da, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,960 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=334949367d92cc18a393737b69358f37, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,960 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac402503c294574c0e555c45a12f0a4e, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,960 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06a1142ff2ac98bd39e6f73547d6aa98, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,960 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4145a9c4106377623484a3eb786fe8a, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,960 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a6bb8489f79432157c419605ac84965, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,960 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c5d2bc20a7b33de1c8b5a53f7fee57, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,960 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=996589dbb2419ea602bc5e348cbd05df, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,960 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cf2d9c6c6cf36bd61bb35f51df8f42, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,960 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b8b8290cc9d27b8d226e5dc2b7d770d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,960 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ef98738dc2598149d89e210ccdb4e6f, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,960 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=317db83a078d27fdfa108a4454669013, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,960 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21af909e3b8cf553daecee4e59c0898b, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,961 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f45cc001ffd649bcfbfe0f7af1630961, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,961 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e9187292024abc7067d470edad3ae67, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,961 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e7efa0e8c09fd1f0e1a765d7fc38181, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,961 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1062179709b27df36edcb7f6b054a4a6, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,961 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T21:49:56,961 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732657856961 2024-11-26T21:49:56,961 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732657916961 2024-11-26T21:49:56,961 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-26T21:49:56,961 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T21:49:56,961 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:56,961 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T21:49:56,961 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T21:49:56,961 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T21:49:56,961 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T21:49:56,962 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T21:49:56,962 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T21:49:56,962 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T21:49:56,964 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T21:49:56,964 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,964 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T21:49:56,964 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T21:49:56,964 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,964 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,964 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T21:49:56,964 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T21:49:56,964 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T21:49:56,965 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T21:49:56,966 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true 2024-11-26T21:49:56,966 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,966 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,966 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T21:49:56,966 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,966 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,966 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,966 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN 2024-11-26T21:49:56,966 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,966 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,966 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,966 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,966 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,966 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,966 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,966 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,966 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN 2024-11-26T21:49:56,967 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-26T21:49:56,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=61 2024-11-26T21:49:56,967 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=828b3f090c91,40359,1732657789839, table=hbase:meta, region=1588230740 2024-11-26T21:49:56,967 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 828b3f090c91,40359,1732657789839, state=OPEN 2024-11-26T21:49:56,969 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,969 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,969 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,969 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,969 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,969 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,969 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c4145a9c4106377623484a3eb786fe8a, ASSIGN to state=OFFLINE, location=null, table=test, region=c4145a9c4106377623484a3eb786fe8a to restore RIT 2024-11-26T21:49:56,970 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732657826970 2024-11-26T21:49:56,970 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T21:49:56,970 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,971 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T21:49:56,972 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T21:49:56,973 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,973 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:56,973 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,975 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf200defbfcf83538e813f9fa91d9c35, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,975 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36699c847bc0e6c6b06fd8533d3c9edb, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,975 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34e003c01eb3f0a2e10e33152402164c, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,975 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a7c374815f888db179beadaf2489ad7, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,975 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=651cf53d9624e6cc98e46d6578c71496, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,975 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0a1ab633a90e6b5f5e9ba8ce6f66010, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,976 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8be22ac0f4cc7a17874b9d10ba204518, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,976 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a2d6b8d1f56582e3f52e3cfe7a116c1d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,976 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d22f8ae31d7e7a4d0c284e3b9676a393, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,976 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdd2d12bbd55678c7f638b4ef0fde3da, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,976 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=334949367d92cc18a393737b69358f37, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,976 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac402503c294574c0e555c45a12f0a4e, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,976 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06a1142ff2ac98bd39e6f73547d6aa98, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,976 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4145a9c4106377623484a3eb786fe8a, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,977 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a6bb8489f79432157c419605ac84965, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,977 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c5d2bc20a7b33de1c8b5a53f7fee57, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,977 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=996589dbb2419ea602bc5e348cbd05df, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,977 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cf2d9c6c6cf36bd61bb35f51df8f42, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,977 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b8b8290cc9d27b8d226e5dc2b7d770d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,977 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ef98738dc2598149d89e210ccdb4e6f, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,977 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=317db83a078d27fdfa108a4454669013, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,977 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21af909e3b8cf553daecee4e59c0898b, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,977 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f45cc001ffd649bcfbfe0f7af1630961, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,978 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e9187292024abc7067d470edad3ae67, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,978 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e7efa0e8c09fd1f0e1a765d7fc38181, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,978 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1062179709b27df36edcb7f6b054a4a6, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,978 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T21:49:56,978 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732657856978 2024-11-26T21:49:56,978 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732657916978 2024-11-26T21:49:56,978 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-26T21:49:56,978 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T21:49:56,978 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:56,978 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T21:49:56,978 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T21:49:56,979 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T21:49:56,979 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T21:49:56,979 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T21:49:56,979 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T21:49:56,980 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T21:49:56,982 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T21:49:56,982 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T21:49:56,982 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T21:49:56,982 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,982 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,982 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T21:49:56,982 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T21:49:56,982 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T21:49:56,983 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T21:49:56,984 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true 2024-11-26T21:49:56,984 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,984 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:56,984 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T21:49:56,984 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,984 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,985 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,985 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:56,985 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,985 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,985 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,985 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,985 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,985 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:56,985 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-26T21:49:56,986 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=61 2024-11-26T21:49:56,986 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=828b3f090c91,40359,1732657789839, table=hbase:meta, region=1588230740 2024-11-26T21:49:56,986 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 828b3f090c91,40359,1732657789839, state=OPEN 2024-11-26T21:49:56,988 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,988 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,988 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:56,988 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,988 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,988 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:56,989 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732657826989 2024-11-26T21:49:56,989 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T21:49:56,989 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,991 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T21:49:56,993 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T21:49:56,993 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 2024-11-26T21:49:56,993 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:56,993 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,994 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf200defbfcf83538e813f9fa91d9c35, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,994 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36699c847bc0e6c6b06fd8533d3c9edb, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,994 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34e003c01eb3f0a2e10e33152402164c, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,994 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a7c374815f888db179beadaf2489ad7, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,994 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=651cf53d9624e6cc98e46d6578c71496, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,994 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0a1ab633a90e6b5f5e9ba8ce6f66010, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,994 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8be22ac0f4cc7a17874b9d10ba204518, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,994 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a2d6b8d1f56582e3f52e3cfe7a116c1d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,994 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d22f8ae31d7e7a4d0c284e3b9676a393, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,995 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdd2d12bbd55678c7f638b4ef0fde3da, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,995 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=334949367d92cc18a393737b69358f37, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,995 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac402503c294574c0e555c45a12f0a4e, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,995 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06a1142ff2ac98bd39e6f73547d6aa98, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,995 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4145a9c4106377623484a3eb786fe8a, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,995 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a6bb8489f79432157c419605ac84965, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,995 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c5d2bc20a7b33de1c8b5a53f7fee57, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,995 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=996589dbb2419ea602bc5e348cbd05df, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,995 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cf2d9c6c6cf36bd61bb35f51df8f42, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,995 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b8b8290cc9d27b8d226e5dc2b7d770d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,995 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ef98738dc2598149d89e210ccdb4e6f, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,995 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=317db83a078d27fdfa108a4454669013, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,996 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21af909e3b8cf553daecee4e59c0898b, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,996 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f45cc001ffd649bcfbfe0f7af1630961, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,996 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e9187292024abc7067d470edad3ae67, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,996 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e7efa0e8c09fd1f0e1a765d7fc38181, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:56,996 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1062179709b27df36edcb7f6b054a4a6, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:56,996 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T21:49:56,996 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732657856996 2024-11-26T21:49:56,996 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732657916996 2024-11-26T21:49:56,996 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-26T21:49:56,996 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T21:49:56,996 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:56,996 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T21:49:56,996 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T21:49:56,996 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T21:49:56,996 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T21:49:56,996 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T21:49:56,997 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T21:49:56,998 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T21:49:56,998 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44203 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:56,999 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T21:49:56,999 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T21:49:56,999 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T21:49:56,999 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:56,999 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:56,999 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T21:49:56,999 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T21:49:57,000 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T21:49:57,000 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true 2024-11-26T21:49:57,000 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:57,000 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:57,000 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T21:49:57,000 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,40359,1732657789839 2024-11-26T21:49:57,000 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 2024-11-26T21:49:57,000 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:57,000 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:57,001 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:57,001 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:57,001 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:57,001 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.1732657791102, worker=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:57,001 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-26T21:49:57,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=61 2024-11-26T21:49:57,001 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=828b3f090c91,40359,1732657789839, table=hbase:meta, region=1588230740 2024-11-26T21:49:57,002 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 828b3f090c91,40359,1732657789839, state=OPEN 2024-11-26T21:49:57,003 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:57,003 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:57,003 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:57,003 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:57,003 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:57,003 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:57,004 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732657827004 2024-11-26T21:49:57,004 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T21:49:57,004 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:57,006 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T21:49:57,007 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T21:49:57,007 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,40359,1732657789839 2024-11-26T21:49:57,007 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:57,007 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,40359,1732657789839 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:57,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf200defbfcf83538e813f9fa91d9c35, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36699c847bc0e6c6b06fd8533d3c9edb, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34e003c01eb3f0a2e10e33152402164c, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a7c374815f888db179beadaf2489ad7, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=651cf53d9624e6cc98e46d6578c71496, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0a1ab633a90e6b5f5e9ba8ce6f66010, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8be22ac0f4cc7a17874b9d10ba204518, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a2d6b8d1f56582e3f52e3cfe7a116c1d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d22f8ae31d7e7a4d0c284e3b9676a393, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdd2d12bbd55678c7f638b4ef0fde3da, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=334949367d92cc18a393737b69358f37, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac402503c294574c0e555c45a12f0a4e, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06a1142ff2ac98bd39e6f73547d6aa98, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4145a9c4106377623484a3eb786fe8a, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a6bb8489f79432157c419605ac84965, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c5d2bc20a7b33de1c8b5a53f7fee57, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=996589dbb2419ea602bc5e348cbd05df, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cf2d9c6c6cf36bd61bb35f51df8f42, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b8b8290cc9d27b8d226e5dc2b7d770d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ef98738dc2598149d89e210ccdb4e6f, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=317db83a078d27fdfa108a4454669013, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21af909e3b8cf553daecee4e59c0898b, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f45cc001ffd649bcfbfe0f7af1630961, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e9187292024abc7067d470edad3ae67, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e7efa0e8c09fd1f0e1a765d7fc38181, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1062179709b27df36edcb7f6b054a4a6, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,011 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T21:49:57,011 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732657857011 2024-11-26T21:49:57,011 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732657917011 2024-11-26T21:49:57,011 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-26T21:49:57,011 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T21:49:57,011 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:57,011 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T21:49:57,011 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T21:49:57,011 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T21:49:57,011 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T21:49:57,011 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T21:49:57,012 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T21:49:57,012 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T21:49:57,014 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T21:49:57,014 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T21:49:57,014 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T21:49:57,014 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:57,014 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T21:49:57,014 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T21:49:57,015 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T21:49:57,015 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true 2024-11-26T21:49:57,016 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:57,016 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:57,016 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T21:49:57,016 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,40359,1732657789839 2024-11-26T21:49:57,016 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:57,016 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:57,016 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:57,016 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:57,016 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,40359,1732657789839 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:57,016 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-26T21:49:57,017 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=61 2024-11-26T21:49:57,017 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=828b3f090c91,40359,1732657789839, table=hbase:meta, region=1588230740 2024-11-26T21:49:57,017 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 828b3f090c91,40359,1732657789839, state=OPEN 2024-11-26T21:49:57,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:57,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:57,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:57,019 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:57,019 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:57,019 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:57,020 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732657827020 2024-11-26T21:49:57,020 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T21:49:57,020 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:57,021 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T21:49:57,022 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T21:49:57,022 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T21:49:57,022 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:57,022 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:57,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf200defbfcf83538e813f9fa91d9c35, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36699c847bc0e6c6b06fd8533d3c9edb, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34e003c01eb3f0a2e10e33152402164c, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a7c374815f888db179beadaf2489ad7, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=651cf53d9624e6cc98e46d6578c71496, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,027 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0a1ab633a90e6b5f5e9ba8ce6f66010, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,027 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8be22ac0f4cc7a17874b9d10ba204518, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,027 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a2d6b8d1f56582e3f52e3cfe7a116c1d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,027 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d22f8ae31d7e7a4d0c284e3b9676a393, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,027 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdd2d12bbd55678c7f638b4ef0fde3da, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,027 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=334949367d92cc18a393737b69358f37, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,027 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac402503c294574c0e555c45a12f0a4e, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,027 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06a1142ff2ac98bd39e6f73547d6aa98, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4145a9c4106377623484a3eb786fe8a, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a6bb8489f79432157c419605ac84965, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c5d2bc20a7b33de1c8b5a53f7fee57, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=996589dbb2419ea602bc5e348cbd05df, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cf2d9c6c6cf36bd61bb35f51df8f42, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b8b8290cc9d27b8d226e5dc2b7d770d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ef98738dc2598149d89e210ccdb4e6f, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=317db83a078d27fdfa108a4454669013, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21af909e3b8cf553daecee4e59c0898b, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f45cc001ffd649bcfbfe0f7af1630961, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e9187292024abc7067d470edad3ae67, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e7efa0e8c09fd1f0e1a765d7fc38181, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1062179709b27df36edcb7f6b054a4a6, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,029 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T21:49:57,029 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732657857029 2024-11-26T21:49:57,029 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732657917029 2024-11-26T21:49:57,029 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-26T21:49:57,029 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T21:49:57,029 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:57,029 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T21:49:57,030 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T21:49:57,030 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T21:49:57,030 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T21:49:57,030 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T21:49:57,030 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T21:49:57,031 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T21:49:57,033 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T21:49:57,033 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T21:49:57,033 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T21:49:57,033 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T21:49:57,034 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T21:49:57,034 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T21:49:57,035 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true 2024-11-26T21:49:57,035 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:57,035 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:57,035 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T21:49:57,035 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:57,035 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:57,035 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:57,035 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:57,035 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-26T21:49:57,036 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=61 2024-11-26T21:49:57,036 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=828b3f090c91,40359,1732657789839, table=hbase:meta, region=1588230740 2024-11-26T21:49:57,037 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 828b3f090c91,40359,1732657789839, state=OPEN 2024-11-26T21:49:57,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:57,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:57,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:57,038 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:57,038 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:57,038 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:57,039 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732657827039 2024-11-26T21:49:57,039 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T21:49:57,039 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:57,041 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T21:49:57,042 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T21:49:57,043 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:57,043 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:57,043 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:57,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf200defbfcf83538e813f9fa91d9c35, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36699c847bc0e6c6b06fd8533d3c9edb, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34e003c01eb3f0a2e10e33152402164c, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,045 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a7c374815f888db179beadaf2489ad7, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,045 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=651cf53d9624e6cc98e46d6578c71496, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,045 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0a1ab633a90e6b5f5e9ba8ce6f66010, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,045 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8be22ac0f4cc7a17874b9d10ba204518, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,045 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a2d6b8d1f56582e3f52e3cfe7a116c1d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,045 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d22f8ae31d7e7a4d0c284e3b9676a393, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,045 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdd2d12bbd55678c7f638b4ef0fde3da, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,045 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=334949367d92cc18a393737b69358f37, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,045 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac402503c294574c0e555c45a12f0a4e, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,045 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06a1142ff2ac98bd39e6f73547d6aa98, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,045 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4145a9c4106377623484a3eb786fe8a, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,045 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a6bb8489f79432157c419605ac84965, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,046 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c5d2bc20a7b33de1c8b5a53f7fee57, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,046 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=996589dbb2419ea602bc5e348cbd05df, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,046 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cf2d9c6c6cf36bd61bb35f51df8f42, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,046 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b8b8290cc9d27b8d226e5dc2b7d770d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,046 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ef98738dc2598149d89e210ccdb4e6f, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,046 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=317db83a078d27fdfa108a4454669013, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,046 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21af909e3b8cf553daecee4e59c0898b, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,046 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f45cc001ffd649bcfbfe0f7af1630961, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,046 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e9187292024abc7067d470edad3ae67, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,046 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e7efa0e8c09fd1f0e1a765d7fc38181, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,046 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1062179709b27df36edcb7f6b054a4a6, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,046 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T21:49:57,046 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732657857046 2024-11-26T21:49:57,046 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732657917046 2024-11-26T21:49:57,047 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-26T21:49:57,047 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T21:49:57,047 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:57,047 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T21:49:57,047 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T21:49:57,047 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T21:49:57,047 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T21:49:57,047 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T21:49:57,047 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T21:49:57,048 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T21:49:57,050 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T21:49:57,050 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T21:49:57,050 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T21:49:57,050 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T21:49:57,051 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T21:49:57,051 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true 2024-11-26T21:49:57,051 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:57,052 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:57,052 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:57,052 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:57,052 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:57,052 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-26T21:49:57,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=61 2024-11-26T21:49:57,053 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=828b3f090c91,40359,1732657789839, table=hbase:meta, region=1588230740 2024-11-26T21:49:57,053 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 828b3f090c91,40359,1732657789839, state=OPEN 2024-11-26T21:49:57,054 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:57,054 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:57,054 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:57,055 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:57,055 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:57,055 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:57,056 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732657827056 2024-11-26T21:49:57,056 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T21:49:57,056 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:57,058 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T21:49:57,059 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T21:49:57,059 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:57,059 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:57,059 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:57,060 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf200defbfcf83538e813f9fa91d9c35, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,060 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36699c847bc0e6c6b06fd8533d3c9edb, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,060 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34e003c01eb3f0a2e10e33152402164c, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,060 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a7c374815f888db179beadaf2489ad7, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,061 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=651cf53d9624e6cc98e46d6578c71496, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,061 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0a1ab633a90e6b5f5e9ba8ce6f66010, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,061 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8be22ac0f4cc7a17874b9d10ba204518, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,061 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a2d6b8d1f56582e3f52e3cfe7a116c1d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,061 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d22f8ae31d7e7a4d0c284e3b9676a393, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,061 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdd2d12bbd55678c7f638b4ef0fde3da, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,061 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=334949367d92cc18a393737b69358f37, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,061 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac402503c294574c0e555c45a12f0a4e, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,061 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06a1142ff2ac98bd39e6f73547d6aa98, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,061 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4145a9c4106377623484a3eb786fe8a, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,061 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a6bb8489f79432157c419605ac84965, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,061 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c5d2bc20a7b33de1c8b5a53f7fee57, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,061 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=996589dbb2419ea602bc5e348cbd05df, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cf2d9c6c6cf36bd61bb35f51df8f42, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b8b8290cc9d27b8d226e5dc2b7d770d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ef98738dc2598149d89e210ccdb4e6f, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=317db83a078d27fdfa108a4454669013, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21af909e3b8cf553daecee4e59c0898b, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f45cc001ffd649bcfbfe0f7af1630961, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e9187292024abc7067d470edad3ae67, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e7efa0e8c09fd1f0e1a765d7fc38181, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1062179709b27df36edcb7f6b054a4a6, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,062 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T21:49:57,062 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732657857062 2024-11-26T21:49:57,062 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732657917062 2024-11-26T21:49:57,062 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-26T21:49:57,062 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T21:49:57,062 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:57,063 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T21:49:57,063 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T21:49:57,063 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T21:49:57,063 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T21:49:57,063 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T21:49:57,063 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T21:49:57,064 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T21:49:57,065 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T21:49:57,065 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T21:49:57,065 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T21:49:57,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T21:49:57,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true 2024-11-26T21:49:57,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 2024-11-26T21:49:57,066 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:57,066 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 828b3f090c91%2C46545%2C1732657789755.meta.1732657791604.meta, worker=828b3f090c91,42273,1732657789877 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T21:49:57,067 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-26T21:49:57,067 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=61 2024-11-26T21:49:57,067 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=828b3f090c91,40359,1732657789839, table=hbase:meta, region=1588230740 2024-11-26T21:49:57,068 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 828b3f090c91,40359,1732657789839, state=OPEN 2024-11-26T21:49:57,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:57,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:57,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:57,069 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:57,069 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:57,069 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:57,071 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732657827070 2024-11-26T21:49:57,071 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T21:49:57,071 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:57,072 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T21:49:57,074 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T21:49:57,074 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true 2024-11-26T21:49:57,074 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:57,075 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:57,075 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf200defbfcf83538e813f9fa91d9c35, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,075 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36699c847bc0e6c6b06fd8533d3c9edb, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,075 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34e003c01eb3f0a2e10e33152402164c, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,075 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a7c374815f888db179beadaf2489ad7, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,075 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=651cf53d9624e6cc98e46d6578c71496, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,076 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0a1ab633a90e6b5f5e9ba8ce6f66010, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,076 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8be22ac0f4cc7a17874b9d10ba204518, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,076 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a2d6b8d1f56582e3f52e3cfe7a116c1d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,076 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d22f8ae31d7e7a4d0c284e3b9676a393, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,076 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdd2d12bbd55678c7f638b4ef0fde3da, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,076 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=334949367d92cc18a393737b69358f37, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,076 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac402503c294574c0e555c45a12f0a4e, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,076 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06a1142ff2ac98bd39e6f73547d6aa98, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,076 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4145a9c4106377623484a3eb786fe8a, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,076 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a6bb8489f79432157c419605ac84965, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,077 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c5d2bc20a7b33de1c8b5a53f7fee57, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,077 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=996589dbb2419ea602bc5e348cbd05df, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,077 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cf2d9c6c6cf36bd61bb35f51df8f42, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,077 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b8b8290cc9d27b8d226e5dc2b7d770d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,077 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ef98738dc2598149d89e210ccdb4e6f, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,077 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=317db83a078d27fdfa108a4454669013, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,077 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21af909e3b8cf553daecee4e59c0898b, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,077 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f45cc001ffd649bcfbfe0f7af1630961, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,077 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e9187292024abc7067d470edad3ae67, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,077 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e7efa0e8c09fd1f0e1a765d7fc38181, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,077 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1062179709b27df36edcb7f6b054a4a6, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,077 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T21:49:57,077 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732657857077 2024-11-26T21:49:57,077 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732657917077 2024-11-26T21:49:57,077 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-26T21:49:57,078 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T21:49:57,078 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:57,078 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T21:49:57,078 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T21:49:57,078 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T21:49:57,078 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T21:49:57,078 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T21:49:57,078 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T21:49:57,079 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T21:49:57,080 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T21:49:57,080 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T21:49:57,081 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T21:49:57,081 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true 2024-11-26T21:49:57,081 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T21:49:57,081 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-26T21:49:57,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=61 2024-11-26T21:49:57,082 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=828b3f090c91,40359,1732657789839, table=hbase:meta, region=1588230740 2024-11-26T21:49:57,082 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 828b3f090c91,40359,1732657789839, state=OPEN 2024-11-26T21:49:57,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:57,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:57,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:49:57,083 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:57,083 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:57,083 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:49:57,085 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732657827085 2024-11-26T21:49:57,085 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T21:49:57,085 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T21:49:57,087 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T21:49:57,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf200defbfcf83538e813f9fa91d9c35, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36699c847bc0e6c6b06fd8533d3c9edb, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34e003c01eb3f0a2e10e33152402164c, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a7c374815f888db179beadaf2489ad7, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,091 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=57, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure 651cf53d9624e6cc98e46d6578c71496, server=828b3f090c91,42273,1732657789877:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 828b3f090c91,46545,1732657789755, splitWal=true, meta=true exec-time=2.6400 sec 2024-11-26T21:49:57,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=651cf53d9624e6cc98e46d6578c71496, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f0a1ab633a90e6b5f5e9ba8ce6f66010, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8be22ac0f4cc7a17874b9d10ba204518, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a2d6b8d1f56582e3f52e3cfe7a116c1d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d22f8ae31d7e7a4d0c284e3b9676a393, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdd2d12bbd55678c7f638b4ef0fde3da, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=334949367d92cc18a393737b69358f37, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac402503c294574c0e555c45a12f0a4e, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06a1142ff2ac98bd39e6f73547d6aa98, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4145a9c4106377623484a3eb786fe8a, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a6bb8489f79432157c419605ac84965, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c5d2bc20a7b33de1c8b5a53f7fee57, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=996589dbb2419ea602bc5e348cbd05df, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30cf2d9c6c6cf36bd61bb35f51df8f42, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b8b8290cc9d27b8d226e5dc2b7d770d, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1ef98738dc2598149d89e210ccdb4e6f, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=317db83a078d27fdfa108a4454669013, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=21af909e3b8cf553daecee4e59c0898b, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f45cc001ffd649bcfbfe0f7af1630961, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e9187292024abc7067d470edad3ae67, regionState=OPEN, lastHost=828b3f090c91,42273,1732657789877, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e7efa0e8c09fd1f0e1a765d7fc38181, regionState=OPENING, lastHost=828b3f090c91,46545,1732657789755, regionLocation=828b3f090c91,42273,1732657789877, openSeqNum=2 2024-11-26T21:49:57,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1062179709b27df36edcb7f6b054a4a6, regionState=OPEN, lastHost=828b3f090c91,40359,1732657789839, regionLocation=828b3f090c91,40359,1732657789839, openSeqNum=2 2024-11-26T21:49:57,093 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T21:49:57,093 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732657857093 2024-11-26T21:49:57,093 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732657917093 2024-11-26T21:49:57,093 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-26T21:49:57,109 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=352 (was 386), OpenFileDescriptor=626 (was 649), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=234 (was 234), ProcessCount=11 (was 11), AvailableMemoryMB=11051 (was 11124) 2024-11-26T21:49:57,109 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-26T21:49:57,110 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T21:49:57,110 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T21:49:57,110 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T21:49:57,110 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T21:49:57,111 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T21:49:57,111 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-26T21:49:57,111 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1166144013, stopped=false 2024-11-26T21:49:57,111 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=828b3f090c91,44203,1732657789166 2024-11-26T21:49:57,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T21:49:57,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T21:49:57,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T21:49:57,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:49:57,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:49:57,113 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:57,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:49:57,113 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T21:49:57,113 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T21:49:57,113 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T21:49:57,113 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T21:49:57,113 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T21:49:57,113 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T21:49:57,114 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '828b3f090c91,40359,1732657789839' ***** 2024-11-26T21:49:57,114 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T21:49:57,114 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '828b3f090c91,42273,1732657789877' ***** 2024-11-26T21:49:57,114 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T21:49:57,114 INFO [RS:1;828b3f090c91:40359 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T21:49:57,114 INFO [RS:2;828b3f090c91:42273 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T21:49:57,114 INFO [RS:1;828b3f090c91:40359 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T21:49:57,114 INFO [RS:1;828b3f090c91:40359 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T21:49:57,114 INFO [RS:2;828b3f090c91:42273 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T21:49:57,115 INFO [RS:2;828b3f090c91:42273 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T21:49:57,115 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T21:49:57,115 INFO [RS:1;828b3f090c91:40359 {}] regionserver.HRegionServer(3091): Received CLOSE for 7a7c374815f888db179beadaf2489ad7 2024-11-26T21:49:57,115 INFO [RS:2;828b3f090c91:42273 {}] regionserver.HRegionServer(3091): Received CLOSE for 996589dbb2419ea602bc5e348cbd05df 2024-11-26T21:49:57,115 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T21:49:57,115 INFO [RS:2;828b3f090c91:42273 {}] regionserver.HRegionServer(3091): Received CLOSE for 34e003c01eb3f0a2e10e33152402164c 2024-11-26T21:49:57,115 INFO [RS:2;828b3f090c91:42273 {}] regionserver.HRegionServer(3091): Received CLOSE for 30cf2d9c6c6cf36bd61bb35f51df8f42 2024-11-26T21:49:57,115 INFO [RS:2;828b3f090c91:42273 {}] regionserver.HRegionServer(3091): Received CLOSE for 1e7efa0e8c09fd1f0e1a765d7fc38181 2024-11-26T21:49:57,115 INFO [RS:2;828b3f090c91:42273 {}] regionserver.HRegionServer(3091): Received CLOSE for d1c5d2bc20a7b33de1c8b5a53f7fee57 2024-11-26T21:49:57,115 INFO [RS:2;828b3f090c91:42273 {}] regionserver.HRegionServer(3091): Received CLOSE for 8be22ac0f4cc7a17874b9d10ba204518 2024-11-26T21:49:57,116 INFO [RS:2;828b3f090c91:42273 {}] regionserver.HRegionServer(3091): Received CLOSE for 317db83a078d27fdfa108a4454669013 2024-11-26T21:49:57,116 INFO [RS:2;828b3f090c91:42273 {}] regionserver.HRegionServer(3091): Received CLOSE for cdd2d12bbd55678c7f638b4ef0fde3da 2024-11-26T21:49:57,116 INFO [RS:2;828b3f090c91:42273 {}] regionserver.HRegionServer(3091): Received CLOSE for 06a1142ff2ac98bd39e6f73547d6aa98 2024-11-26T21:49:57,116 INFO [RS:2;828b3f090c91:42273 {}] regionserver.HRegionServer(3091): Received CLOSE for 334949367d92cc18a393737b69358f37 2024-11-26T21:49:57,116 INFO [RS:2;828b3f090c91:42273 {}] regionserver.HRegionServer(3091): Received CLOSE for 0e9187292024abc7067d470edad3ae67 2024-11-26T21:49:57,116 INFO [RS:2;828b3f090c91:42273 {}] regionserver.HRegionServer(3091): Received CLOSE for 651cf53d9624e6cc98e46d6578c71496 2024-11-26T21:49:57,116 INFO [RS:2;828b3f090c91:42273 {}] regionserver.HRegionServer(3091): Received CLOSE for 2b8b8290cc9d27b8d226e5dc2b7d770d 2024-11-26T21:49:57,116 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 996589dbb2419ea602bc5e348cbd05df, disabling compactions & flushes 2024-11-26T21:49:57,116 INFO [RS:2;828b3f090c91:42273 {}] regionserver.HRegionServer(959): stopping server 828b3f090c91,42273,1732657789877 2024-11-26T21:49:57,116 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1732657792023.996589dbb2419ea602bc5e348cbd05df. 2024-11-26T21:49:57,116 INFO [RS:2;828b3f090c91:42273 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T21:49:57,116 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732657792023.996589dbb2419ea602bc5e348cbd05df. 2024-11-26T21:49:57,116 INFO [RS:2;828b3f090c91:42273 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;828b3f090c91:42273. 2024-11-26T21:49:57,116 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732657792023.996589dbb2419ea602bc5e348cbd05df. after waiting 0 ms 2024-11-26T21:49:57,116 INFO [RS:1;828b3f090c91:40359 {}] regionserver.HRegionServer(3091): Received CLOSE for d22f8ae31d7e7a4d0c284e3b9676a393 2024-11-26T21:49:57,116 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732657792023.996589dbb2419ea602bc5e348cbd05df. 2024-11-26T21:49:57,116 INFO [RS:1;828b3f090c91:40359 {}] regionserver.HRegionServer(3091): Received CLOSE for 1062179709b27df36edcb7f6b054a4a6 2024-11-26T21:49:57,116 DEBUG [RS:2;828b3f090c91:42273 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T21:49:57,116 INFO [RS:1;828b3f090c91:40359 {}] regionserver.HRegionServer(3091): Received CLOSE for 36699c847bc0e6c6b06fd8533d3c9edb 2024-11-26T21:49:57,116 DEBUG [RS:2;828b3f090c91:42273 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T21:49:57,116 INFO [RS:1;828b3f090c91:40359 {}] regionserver.HRegionServer(3091): Received CLOSE for 8a6bb8489f79432157c419605ac84965 2024-11-26T21:49:57,116 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7a7c374815f888db179beadaf2489ad7, disabling compactions & flushes 2024-11-26T21:49:57,116 INFO [RS:1;828b3f090c91:40359 {}] regionserver.HRegionServer(3091): Received CLOSE for f45cc001ffd649bcfbfe0f7af1630961 2024-11-26T21:49:57,116 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1732657792023.7a7c374815f888db179beadaf2489ad7. 2024-11-26T21:49:57,117 INFO [RS:1;828b3f090c91:40359 {}] regionserver.HRegionServer(3091): Received CLOSE for 1ef98738dc2598149d89e210ccdb4e6f 2024-11-26T21:49:57,117 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1732657792023.7a7c374815f888db179beadaf2489ad7. 2024-11-26T21:49:57,117 INFO [RS:1;828b3f090c91:40359 {}] regionserver.HRegionServer(3091): Received CLOSE for f0a1ab633a90e6b5f5e9ba8ce6f66010 2024-11-26T21:49:57,117 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1732657792023.7a7c374815f888db179beadaf2489ad7. after waiting 0 ms 2024-11-26T21:49:57,117 INFO [RS:1;828b3f090c91:40359 {}] regionserver.HRegionServer(3091): Received CLOSE for ac402503c294574c0e555c45a12f0a4e 2024-11-26T21:49:57,117 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1732657792023.7a7c374815f888db179beadaf2489ad7. 2024-11-26T21:49:57,117 INFO [RS:1;828b3f090c91:40359 {}] regionserver.HRegionServer(959): stopping server 828b3f090c91,40359,1732657789839 2024-11-26T21:49:57,117 INFO [RS:2;828b3f090c91:42273 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-11-26T21:49:57,117 INFO [RS:1;828b3f090c91:40359 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T21:49:57,117 INFO [RS:1;828b3f090c91:40359 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;828b3f090c91:40359. 2024-11-26T21:49:57,117 DEBUG [RS:1;828b3f090c91:40359 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T21:49:57,117 DEBUG [RS:1;828b3f090c91:40359 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T21:49:57,117 DEBUG [RS:2;828b3f090c91:42273 {}] regionserver.HRegionServer(1351): Waiting on 06a1142ff2ac98bd39e6f73547d6aa98, 0e9187292024abc7067d470edad3ae67, 1e7efa0e8c09fd1f0e1a765d7fc38181, 2b8b8290cc9d27b8d226e5dc2b7d770d, 30cf2d9c6c6cf36bd61bb35f51df8f42, 317db83a078d27fdfa108a4454669013, 334949367d92cc18a393737b69358f37, 34e003c01eb3f0a2e10e33152402164c, 651cf53d9624e6cc98e46d6578c71496, 8be22ac0f4cc7a17874b9d10ba204518, 996589dbb2419ea602bc5e348cbd05df, cdd2d12bbd55678c7f638b4ef0fde3da, d1c5d2bc20a7b33de1c8b5a53f7fee57 2024-11-26T21:49:57,117 INFO [RS:1;828b3f090c91:40359 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T21:49:57,118 INFO [RS:1;828b3f090c91:40359 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T21:49:57,118 INFO [RS:1;828b3f090c91:40359 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T21:49:57,118 INFO [RS:1;828b3f090c91:40359 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-26T21:49:57,121 INFO [RS:1;828b3f090c91:40359 {}] regionserver.HRegionServer(1321): Waiting on 10 regions to close 2024-11-26T21:49:57,121 DEBUG [RS:1;828b3f090c91:40359 {}] regionserver.HRegionServer(1351): Waiting on 1062179709b27df36edcb7f6b054a4a6, 1588230740, 1ef98738dc2598149d89e210ccdb4e6f, 36699c847bc0e6c6b06fd8533d3c9edb, 7a7c374815f888db179beadaf2489ad7, 8a6bb8489f79432157c419605ac84965, ac402503c294574c0e555c45a12f0a4e, bf200defbfcf83538e813f9fa91d9c35, d22f8ae31d7e7a4d0c284e3b9676a393, f0a1ab633a90e6b5f5e9ba8ce6f66010, f45cc001ffd649bcfbfe0f7af1630961 2024-11-26T21:49:57,122 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T21:49:57,122 INFO [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T21:49:57,122 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T21:49:57,122 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T21:49:57,122 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T21:49:57,123 INFO [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.64 KB heapSize=5.42 KB 2024-11-26T21:49:57,123 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/996589dbb2419ea602bc5e348cbd05df/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T21:49:57,125 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/7a7c374815f888db179beadaf2489ad7/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T21:49:57,126 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1732657792023.996589dbb2419ea602bc5e348cbd05df. 2024-11-26T21:49:57,126 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 996589dbb2419ea602bc5e348cbd05df: Waiting for close lock at 1732657797115Running coprocessor pre-close hooks at 1732657797116 (+1 ms)Disabling compacts and flushes for region at 1732657797116Disabling writes for close at 1732657797116Writing region close event to WAL at 1732657797118 (+2 ms)Running coprocessor post-close hooks at 1732657797124 (+6 ms)Closed at 1732657797126 (+2 ms) 2024-11-26T21:49:57,126 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1732657792023.7a7c374815f888db179beadaf2489ad7. 2024-11-26T21:49:57,126 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7a7c374815f888db179beadaf2489ad7: Waiting for close lock at 1732657797116Running coprocessor pre-close hooks at 1732657797116Disabling compacts and flushes for region at 1732657797116Disabling writes for close at 1732657797117 (+1 ms)Writing region close event to WAL at 1732657797118 (+1 ms)Running coprocessor post-close hooks at 1732657797126 (+8 ms)Closed at 1732657797126 2024-11-26T21:49:57,127 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1732657792023.7a7c374815f888db179beadaf2489ad7. 2024-11-26T21:49:57,127 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1732657792023.996589dbb2419ea602bc5e348cbd05df. 2024-11-26T21:49:57,127 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d22f8ae31d7e7a4d0c284e3b9676a393, disabling compactions & flushes 2024-11-26T21:49:57,127 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 34e003c01eb3f0a2e10e33152402164c, disabling compactions & flushes 2024-11-26T21:49:57,127 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1732657792023.d22f8ae31d7e7a4d0c284e3b9676a393. 2024-11-26T21:49:57,127 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1732657792023.34e003c01eb3f0a2e10e33152402164c. 2024-11-26T21:49:57,127 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1732657792023.d22f8ae31d7e7a4d0c284e3b9676a393. 2024-11-26T21:49:57,127 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1732657792023.34e003c01eb3f0a2e10e33152402164c. 2024-11-26T21:49:57,127 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1732657792023.d22f8ae31d7e7a4d0c284e3b9676a393. after waiting 0 ms 2024-11-26T21:49:57,127 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1732657792023.34e003c01eb3f0a2e10e33152402164c. after waiting 0 ms 2024-11-26T21:49:57,127 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1732657792023.d22f8ae31d7e7a4d0c284e3b9676a393. 2024-11-26T21:49:57,127 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1732657792023.34e003c01eb3f0a2e10e33152402164c. 2024-11-26T21:49:57,136 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/d22f8ae31d7e7a4d0c284e3b9676a393/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T21:49:57,136 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/34e003c01eb3f0a2e10e33152402164c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T21:49:57,137 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1732657792023.34e003c01eb3f0a2e10e33152402164c. 2024-11-26T21:49:57,137 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 34e003c01eb3f0a2e10e33152402164c: Waiting for close lock at 1732657797127Running coprocessor pre-close hooks at 1732657797127Disabling compacts and flushes for region at 1732657797127Disabling writes for close at 1732657797127Writing region close event to WAL at 1732657797128 (+1 ms)Running coprocessor post-close hooks at 1732657797137 (+9 ms)Closed at 1732657797137 2024-11-26T21:49:57,137 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1732657792023.d22f8ae31d7e7a4d0c284e3b9676a393. 2024-11-26T21:49:57,137 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d22f8ae31d7e7a4d0c284e3b9676a393: Waiting for close lock at 1732657797127Running coprocessor pre-close hooks at 1732657797127Disabling compacts and flushes for region at 1732657797127Disabling writes for close at 1732657797127Writing region close event to WAL at 1732657797127Running coprocessor post-close hooks at 1732657797136 (+9 ms)Closed at 1732657797137 (+1 ms) 2024-11-26T21:49:57,137 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1732657792023.34e003c01eb3f0a2e10e33152402164c. 2024-11-26T21:49:57,137 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1732657792023.d22f8ae31d7e7a4d0c284e3b9676a393. 2024-11-26T21:49:57,137 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1062179709b27df36edcb7f6b054a4a6, disabling compactions & flushes 2024-11-26T21:49:57,137 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 30cf2d9c6c6cf36bd61bb35f51df8f42, disabling compactions & flushes 2024-11-26T21:49:57,137 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1732657792023.1062179709b27df36edcb7f6b054a4a6. 2024-11-26T21:49:57,137 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1732657792023.30cf2d9c6c6cf36bd61bb35f51df8f42. 2024-11-26T21:49:57,137 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1732657792023.1062179709b27df36edcb7f6b054a4a6. 2024-11-26T21:49:57,137 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732657792023.30cf2d9c6c6cf36bd61bb35f51df8f42. 2024-11-26T21:49:57,137 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1732657792023.1062179709b27df36edcb7f6b054a4a6. after waiting 0 ms 2024-11-26T21:49:57,137 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732657792023.30cf2d9c6c6cf36bd61bb35f51df8f42. after waiting 0 ms 2024-11-26T21:49:57,137 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1732657792023.1062179709b27df36edcb7f6b054a4a6. 2024-11-26T21:49:57,137 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732657792023.30cf2d9c6c6cf36bd61bb35f51df8f42. 2024-11-26T21:49:57,139 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/hbase/meta/1588230740/.tmp/info/fa664ea6c77c441b86cee5eab39d2117 is 138, key is test,eee,1732657792023.651cf53d9624e6cc98e46d6578c71496./info:regioninfo/1732657796259/Put/seqid=0 2024-11-26T21:49:57,142 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/30cf2d9c6c6cf36bd61bb35f51df8f42/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T21:49:57,144 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1732657792023.30cf2d9c6c6cf36bd61bb35f51df8f42. 2024-11-26T21:49:57,144 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 30cf2d9c6c6cf36bd61bb35f51df8f42: Waiting for close lock at 1732657797137Running coprocessor pre-close hooks at 1732657797137Disabling compacts and flushes for region at 1732657797137Disabling writes for close at 1732657797137Writing region close event to WAL at 1732657797138 (+1 ms)Running coprocessor post-close hooks at 1732657797144 (+6 ms)Closed at 1732657797144 2024-11-26T21:49:57,144 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1732657792023.30cf2d9c6c6cf36bd61bb35f51df8f42. 2024-11-26T21:49:57,144 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/1062179709b27df36edcb7f6b054a4a6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T21:49:57,144 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1e7efa0e8c09fd1f0e1a765d7fc38181, disabling compactions & flushes 2024-11-26T21:49:57,145 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1732657792023.1e7efa0e8c09fd1f0e1a765d7fc38181. 2024-11-26T21:49:57,145 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732657792023.1e7efa0e8c09fd1f0e1a765d7fc38181. 2024-11-26T21:49:57,145 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732657792023.1e7efa0e8c09fd1f0e1a765d7fc38181. after waiting 0 ms 2024-11-26T21:49:57,145 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732657792023.1e7efa0e8c09fd1f0e1a765d7fc38181. 2024-11-26T21:49:57,145 INFO [regionserver/828b3f090c91:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T21:49:57,145 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1732657792023.1062179709b27df36edcb7f6b054a4a6. 2024-11-26T21:49:57,145 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1062179709b27df36edcb7f6b054a4a6: Waiting for close lock at 1732657797137Running coprocessor pre-close hooks at 1732657797137Disabling compacts and flushes for region at 1732657797137Disabling writes for close at 1732657797137Writing region close event to WAL at 1732657797138 (+1 ms)Running coprocessor post-close hooks at 1732657797145 (+7 ms)Closed at 1732657797145 2024-11-26T21:49:57,146 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1732657792023.1062179709b27df36edcb7f6b054a4a6. 2024-11-26T21:49:57,146 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 36699c847bc0e6c6b06fd8533d3c9edb, disabling compactions & flushes 2024-11-26T21:49:57,146 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1732657792023.36699c847bc0e6c6b06fd8533d3c9edb. 2024-11-26T21:49:57,146 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1732657792023.36699c847bc0e6c6b06fd8533d3c9edb. 2024-11-26T21:49:57,146 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1732657792023.36699c847bc0e6c6b06fd8533d3c9edb. after waiting 0 ms 2024-11-26T21:49:57,146 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1732657792023.36699c847bc0e6c6b06fd8533d3c9edb. 2024-11-26T21:49:57,146 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T21:49:57,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741869_1045 (size=8177) 2024-11-26T21:49:57,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741869_1045 (size=8177) 2024-11-26T21:49:57,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741869_1045 (size=8177) 2024-11-26T21:49:57,156 INFO [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.64 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/hbase/meta/1588230740/.tmp/info/fa664ea6c77c441b86cee5eab39d2117 2024-11-26T21:49:57,157 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/1e7efa0e8c09fd1f0e1a765d7fc38181/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T21:49:57,157 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1732657792023.1e7efa0e8c09fd1f0e1a765d7fc38181. 2024-11-26T21:49:57,158 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1e7efa0e8c09fd1f0e1a765d7fc38181: Waiting for close lock at 1732657797144Running coprocessor pre-close hooks at 1732657797144Disabling compacts and flushes for region at 1732657797144Disabling writes for close at 1732657797145 (+1 ms)Writing region close event to WAL at 1732657797149 (+4 ms)Running coprocessor post-close hooks at 1732657797157 (+8 ms)Closed at 1732657797157 2024-11-26T21:49:57,158 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1732657792023.1e7efa0e8c09fd1f0e1a765d7fc38181. 2024-11-26T21:49:57,158 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d1c5d2bc20a7b33de1c8b5a53f7fee57, disabling compactions & flushes 2024-11-26T21:49:57,158 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ppp,1732657792023.d1c5d2bc20a7b33de1c8b5a53f7fee57. 2024-11-26T21:49:57,158 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732657792023.d1c5d2bc20a7b33de1c8b5a53f7fee57. 2024-11-26T21:49:57,158 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732657792023.d1c5d2bc20a7b33de1c8b5a53f7fee57. after waiting 0 ms 2024-11-26T21:49:57,158 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732657792023.d1c5d2bc20a7b33de1c8b5a53f7fee57. 2024-11-26T21:49:57,159 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/36699c847bc0e6c6b06fd8533d3c9edb/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T21:49:57,160 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1732657792023.36699c847bc0e6c6b06fd8533d3c9edb. 2024-11-26T21:49:57,160 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 36699c847bc0e6c6b06fd8533d3c9edb: Waiting for close lock at 1732657797146Running coprocessor pre-close hooks at 1732657797146Disabling compacts and flushes for region at 1732657797146Disabling writes for close at 1732657797146Writing region close event to WAL at 1732657797149 (+3 ms)Running coprocessor post-close hooks at 1732657797160 (+11 ms)Closed at 1732657797160 2024-11-26T21:49:57,160 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1732657792023.36699c847bc0e6c6b06fd8533d3c9edb. 2024-11-26T21:49:57,160 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8a6bb8489f79432157c419605ac84965, disabling compactions & flushes 2024-11-26T21:49:57,160 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1732657792023.8a6bb8489f79432157c419605ac84965. 2024-11-26T21:49:57,160 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1732657792023.8a6bb8489f79432157c419605ac84965. 2024-11-26T21:49:57,160 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1732657792023.8a6bb8489f79432157c419605ac84965. after waiting 0 ms 2024-11-26T21:49:57,160 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1732657792023.8a6bb8489f79432157c419605ac84965. 2024-11-26T21:49:57,165 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/d1c5d2bc20a7b33de1c8b5a53f7fee57/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T21:49:57,168 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ppp,1732657792023.d1c5d2bc20a7b33de1c8b5a53f7fee57. 2024-11-26T21:49:57,168 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d1c5d2bc20a7b33de1c8b5a53f7fee57: Waiting for close lock at 1732657797158Running coprocessor pre-close hooks at 1732657797158Disabling compacts and flushes for region at 1732657797158Disabling writes for close at 1732657797158Writing region close event to WAL at 1732657797158Running coprocessor post-close hooks at 1732657797167 (+9 ms)Closed at 1732657797167 2024-11-26T21:49:57,168 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ppp,1732657792023.d1c5d2bc20a7b33de1c8b5a53f7fee57. 2024-11-26T21:49:57,168 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/hbase/meta/1588230740/.tmp/info/fa664ea6c77c441b86cee5eab39d2117 as hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/hbase/meta/1588230740/info/fa664ea6c77c441b86cee5eab39d2117 2024-11-26T21:49:57,168 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8be22ac0f4cc7a17874b9d10ba204518, disabling compactions & flushes 2024-11-26T21:49:57,168 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1732657792023.8be22ac0f4cc7a17874b9d10ba204518. 2024-11-26T21:49:57,168 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732657792023.8be22ac0f4cc7a17874b9d10ba204518. 2024-11-26T21:49:57,168 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732657792023.8be22ac0f4cc7a17874b9d10ba204518. after waiting 0 ms 2024-11-26T21:49:57,168 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732657792023.8be22ac0f4cc7a17874b9d10ba204518. 2024-11-26T21:49:57,169 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/8a6bb8489f79432157c419605ac84965/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T21:49:57,170 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1732657792023.8a6bb8489f79432157c419605ac84965. 2024-11-26T21:49:57,170 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8a6bb8489f79432157c419605ac84965: Waiting for close lock at 1732657797160Running coprocessor pre-close hooks at 1732657797160Disabling compacts and flushes for region at 1732657797160Disabling writes for close at 1732657797160Writing region close event to WAL at 1732657797161 (+1 ms)Running coprocessor post-close hooks at 1732657797169 (+8 ms)Closed at 1732657797170 (+1 ms) 2024-11-26T21:49:57,170 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1732657792023.8a6bb8489f79432157c419605ac84965. 2024-11-26T21:49:57,170 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f45cc001ffd649bcfbfe0f7af1630961, disabling compactions & flushes 2024-11-26T21:49:57,171 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1732657792023.f45cc001ffd649bcfbfe0f7af1630961. 2024-11-26T21:49:57,171 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1732657792023.f45cc001ffd649bcfbfe0f7af1630961. 2024-11-26T21:49:57,171 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1732657792023.f45cc001ffd649bcfbfe0f7af1630961. after waiting 0 ms 2024-11-26T21:49:57,171 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1732657792023.f45cc001ffd649bcfbfe0f7af1630961. 2024-11-26T21:49:57,176 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/8be22ac0f4cc7a17874b9d10ba204518/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T21:49:57,179 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1732657792023.8be22ac0f4cc7a17874b9d10ba204518. 2024-11-26T21:49:57,179 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8be22ac0f4cc7a17874b9d10ba204518: Waiting for close lock at 1732657797168Running coprocessor pre-close hooks at 1732657797168Disabling compacts and flushes for region at 1732657797168Disabling writes for close at 1732657797168Writing region close event to WAL at 1732657797169 (+1 ms)Running coprocessor post-close hooks at 1732657797179 (+10 ms)Closed at 1732657797179 2024-11-26T21:49:57,179 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/f45cc001ffd649bcfbfe0f7af1630961/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T21:49:57,179 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1732657792023.8be22ac0f4cc7a17874b9d10ba204518. 2024-11-26T21:49:57,179 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 317db83a078d27fdfa108a4454669013, disabling compactions & flushes 2024-11-26T21:49:57,179 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1732657792023.317db83a078d27fdfa108a4454669013. 2024-11-26T21:49:57,179 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732657792023.317db83a078d27fdfa108a4454669013. 2024-11-26T21:49:57,179 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732657792023.317db83a078d27fdfa108a4454669013. after waiting 0 ms 2024-11-26T21:49:57,179 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732657792023.317db83a078d27fdfa108a4454669013. 2024-11-26T21:49:57,180 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1732657792023.f45cc001ffd649bcfbfe0f7af1630961. 2024-11-26T21:49:57,180 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f45cc001ffd649bcfbfe0f7af1630961: Waiting for close lock at 1732657797170Running coprocessor pre-close hooks at 1732657797170Disabling compacts and flushes for region at 1732657797170Disabling writes for close at 1732657797171 (+1 ms)Writing region close event to WAL at 1732657797171Running coprocessor post-close hooks at 1732657797180 (+9 ms)Closed at 1732657797180 2024-11-26T21:49:57,180 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1732657792023.f45cc001ffd649bcfbfe0f7af1630961. 2024-11-26T21:49:57,180 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1ef98738dc2598149d89e210ccdb4e6f, disabling compactions & flushes 2024-11-26T21:49:57,180 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1732657792023.1ef98738dc2598149d89e210ccdb4e6f. 2024-11-26T21:49:57,180 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732657792023.1ef98738dc2598149d89e210ccdb4e6f. 2024-11-26T21:49:57,180 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732657792023.1ef98738dc2598149d89e210ccdb4e6f. after waiting 0 ms 2024-11-26T21:49:57,181 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732657792023.1ef98738dc2598149d89e210ccdb4e6f. 2024-11-26T21:49:57,181 INFO [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/hbase/meta/1588230740/info/fa664ea6c77c441b86cee5eab39d2117, entries=24, sequenceid=71, filesize=8.0 K 2024-11-26T21:49:57,183 INFO [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.64 KB/2708, heapSize ~4.66 KB/4768, currentSize=0 B/0 for 1588230740 in 60ms, sequenceid=71, compaction requested=false 2024-11-26T21:49:57,190 INFO [regionserver/828b3f090c91:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T21:49:57,194 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/317db83a078d27fdfa108a4454669013/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T21:49:57,195 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1732657792023.317db83a078d27fdfa108a4454669013. 2024-11-26T21:49:57,195 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 317db83a078d27fdfa108a4454669013: Waiting for close lock at 1732657797179Running coprocessor pre-close hooks at 1732657797179Disabling compacts and flushes for region at 1732657797179Disabling writes for close at 1732657797179Writing region close event to WAL at 1732657797181 (+2 ms)Running coprocessor post-close hooks at 1732657797194 (+13 ms)Closed at 1732657797194 2024-11-26T21:49:57,195 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1732657792023.317db83a078d27fdfa108a4454669013. 2024-11-26T21:49:57,195 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cdd2d12bbd55678c7f638b4ef0fde3da, disabling compactions & flushes 2024-11-26T21:49:57,195 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1732657792023.cdd2d12bbd55678c7f638b4ef0fde3da. 2024-11-26T21:49:57,195 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732657792023.cdd2d12bbd55678c7f638b4ef0fde3da. 2024-11-26T21:49:57,195 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732657792023.cdd2d12bbd55678c7f638b4ef0fde3da. after waiting 0 ms 2024-11-26T21:49:57,195 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732657792023.cdd2d12bbd55678c7f638b4ef0fde3da. 2024-11-26T21:49:57,198 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/1ef98738dc2598149d89e210ccdb4e6f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T21:49:57,198 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1732657792023.1ef98738dc2598149d89e210ccdb4e6f. 2024-11-26T21:49:57,198 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1ef98738dc2598149d89e210ccdb4e6f: Waiting for close lock at 1732657797180Running coprocessor pre-close hooks at 1732657797180Disabling compacts and flushes for region at 1732657797180Disabling writes for close at 1732657797180Writing region close event to WAL at 1732657797185 (+5 ms)Running coprocessor post-close hooks at 1732657797198 (+13 ms)Closed at 1732657797198 2024-11-26T21:49:57,199 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1732657792023.1ef98738dc2598149d89e210ccdb4e6f. 2024-11-26T21:49:57,199 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f0a1ab633a90e6b5f5e9ba8ce6f66010, disabling compactions & flushes 2024-11-26T21:49:57,199 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1732657792023.f0a1ab633a90e6b5f5e9ba8ce6f66010. 2024-11-26T21:49:57,199 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1732657792023.f0a1ab633a90e6b5f5e9ba8ce6f66010. 2024-11-26T21:49:57,199 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1732657792023.f0a1ab633a90e6b5f5e9ba8ce6f66010. after waiting 0 ms 2024-11-26T21:49:57,199 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1732657792023.f0a1ab633a90e6b5f5e9ba8ce6f66010. 2024-11-26T21:49:57,204 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/cdd2d12bbd55678c7f638b4ef0fde3da/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T21:49:57,206 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1732657792023.cdd2d12bbd55678c7f638b4ef0fde3da. 2024-11-26T21:49:57,206 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cdd2d12bbd55678c7f638b4ef0fde3da: Waiting for close lock at 1732657797195Running coprocessor pre-close hooks at 1732657797195Disabling compacts and flushes for region at 1732657797195Disabling writes for close at 1732657797195Writing region close event to WAL at 1732657797196 (+1 ms)Running coprocessor post-close hooks at 1732657797206 (+10 ms)Closed at 1732657797206 2024-11-26T21:49:57,206 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1732657792023.cdd2d12bbd55678c7f638b4ef0fde3da. 2024-11-26T21:49:57,206 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 06a1142ff2ac98bd39e6f73547d6aa98, disabling compactions & flushes 2024-11-26T21:49:57,206 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1732657792023.06a1142ff2ac98bd39e6f73547d6aa98. 2024-11-26T21:49:57,206 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1732657792023.06a1142ff2ac98bd39e6f73547d6aa98. 2024-11-26T21:49:57,207 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1732657792023.06a1142ff2ac98bd39e6f73547d6aa98. after waiting 0 ms 2024-11-26T21:49:57,207 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1732657792023.06a1142ff2ac98bd39e6f73547d6aa98. 2024-11-26T21:49:57,210 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/f0a1ab633a90e6b5f5e9ba8ce6f66010/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T21:49:57,211 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/hbase/meta/1588230740/recovered.edits/74.seqid, newMaxSeqId=74, maxSeqId=60 2024-11-26T21:49:57,211 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1732657792023.f0a1ab633a90e6b5f5e9ba8ce6f66010. 2024-11-26T21:49:57,211 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f0a1ab633a90e6b5f5e9ba8ce6f66010: Waiting for close lock at 1732657797199Running coprocessor pre-close hooks at 1732657797199Disabling compacts and flushes for region at 1732657797199Disabling writes for close at 1732657797199Writing region close event to WAL at 1732657797201 (+2 ms)Running coprocessor post-close hooks at 1732657797211 (+10 ms)Closed at 1732657797211 2024-11-26T21:49:57,211 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1732657792023.f0a1ab633a90e6b5f5e9ba8ce6f66010. 2024-11-26T21:49:57,211 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ac402503c294574c0e555c45a12f0a4e, disabling compactions & flushes 2024-11-26T21:49:57,211 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1732657792023.ac402503c294574c0e555c45a12f0a4e. 2024-11-26T21:49:57,211 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1732657792023.ac402503c294574c0e555c45a12f0a4e. 2024-11-26T21:49:57,211 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1732657792023.ac402503c294574c0e555c45a12f0a4e. after waiting 0 ms 2024-11-26T21:49:57,211 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1732657792023.ac402503c294574c0e555c45a12f0a4e. 2024-11-26T21:49:57,212 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T21:49:57,212 INFO [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T21:49:57,212 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732657797122Running coprocessor pre-close hooks at 1732657797122Disabling compacts and flushes for region at 1732657797122Disabling writes for close at 1732657797122Obtaining lock to block concurrent updates at 1732657797123 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732657797123Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2708, getHeapSize=5488, getOffHeapSize=0, getCellsCount=24 at 1732657797132 (+9 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732657797133 (+1 ms)Flushing 1588230740/info: creating writer at 1732657797133Flushing 1588230740/info: appending metadata at 1732657797139 (+6 ms)Flushing 1588230740/info: closing flushed file at 1732657797139Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c2bd52f: reopening flushed file at 1732657797166 (+27 ms)Finished flush of dataSize ~2.64 KB/2708, heapSize ~4.66 KB/4768, currentSize=0 B/0 for 1588230740 in 60ms, sequenceid=71, compaction requested=false at 1732657797183 (+17 ms)Writing region close event to WAL at 1732657797201 (+18 ms)Running coprocessor post-close hooks at 1732657797212 (+11 ms)Closed at 1732657797212 2024-11-26T21:49:57,212 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-26T21:49:57,218 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/06a1142ff2ac98bd39e6f73547d6aa98/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T21:49:57,218 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/ac402503c294574c0e555c45a12f0a4e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T21:49:57,219 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1732657792023.06a1142ff2ac98bd39e6f73547d6aa98. 2024-11-26T21:49:57,219 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1732657792023.ac402503c294574c0e555c45a12f0a4e. 2024-11-26T21:49:57,219 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 06a1142ff2ac98bd39e6f73547d6aa98: Waiting for close lock at 1732657797206Running coprocessor pre-close hooks at 1732657797206Disabling compacts and flushes for region at 1732657797206Disabling writes for close at 1732657797207 (+1 ms)Writing region close event to WAL at 1732657797207Running coprocessor post-close hooks at 1732657797219 (+12 ms)Closed at 1732657797219 2024-11-26T21:49:57,219 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ac402503c294574c0e555c45a12f0a4e: Waiting for close lock at 1732657797211Running coprocessor pre-close hooks at 1732657797211Disabling compacts and flushes for region at 1732657797211Disabling writes for close at 1732657797211Writing region close event to WAL at 1732657797212 (+1 ms)Running coprocessor post-close hooks at 1732657797219 (+7 ms)Closed at 1732657797219 2024-11-26T21:49:57,219 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1732657792023.06a1142ff2ac98bd39e6f73547d6aa98. 2024-11-26T21:49:57,219 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1732657792023.ac402503c294574c0e555c45a12f0a4e. 2024-11-26T21:49:57,219 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 334949367d92cc18a393737b69358f37, disabling compactions & flushes 2024-11-26T21:49:57,219 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1732657792023.334949367d92cc18a393737b69358f37. 2024-11-26T21:49:57,219 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1732657792023.334949367d92cc18a393737b69358f37. 2024-11-26T21:49:57,219 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1732657792023.334949367d92cc18a393737b69358f37. after waiting 0 ms 2024-11-26T21:49:57,219 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1732657792023.334949367d92cc18a393737b69358f37. 2024-11-26T21:49:57,226 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/334949367d92cc18a393737b69358f37/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T21:49:57,227 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1732657792023.334949367d92cc18a393737b69358f37. 2024-11-26T21:49:57,227 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 334949367d92cc18a393737b69358f37: Waiting for close lock at 1732657797219Running coprocessor pre-close hooks at 1732657797219Disabling compacts and flushes for region at 1732657797219Disabling writes for close at 1732657797219Writing region close event to WAL at 1732657797220 (+1 ms)Running coprocessor post-close hooks at 1732657797227 (+7 ms)Closed at 1732657797227 2024-11-26T21:49:57,227 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1732657792023.334949367d92cc18a393737b69358f37. 2024-11-26T21:49:57,227 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0e9187292024abc7067d470edad3ae67, disabling compactions & flushes 2024-11-26T21:49:57,227 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1732657792023.0e9187292024abc7067d470edad3ae67. 2024-11-26T21:49:57,227 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1732657792023.0e9187292024abc7067d470edad3ae67. 2024-11-26T21:49:57,227 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1732657792023.0e9187292024abc7067d470edad3ae67. after waiting 0 ms 2024-11-26T21:49:57,227 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1732657792023.0e9187292024abc7067d470edad3ae67. 2024-11-26T21:49:57,229 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-11-26T21:49:57,233 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/0e9187292024abc7067d470edad3ae67/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T21:49:57,233 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1732657792023.0e9187292024abc7067d470edad3ae67. 2024-11-26T21:49:57,233 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0e9187292024abc7067d470edad3ae67: Waiting for close lock at 1732657797227Running coprocessor pre-close hooks at 1732657797227Disabling compacts and flushes for region at 1732657797227Disabling writes for close at 1732657797227Writing region close event to WAL at 1732657797228 (+1 ms)Running coprocessor post-close hooks at 1732657797233 (+5 ms)Closed at 1732657797233 2024-11-26T21:49:57,234 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1732657792023.0e9187292024abc7067d470edad3ae67. 2024-11-26T21:49:57,234 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 651cf53d9624e6cc98e46d6578c71496, disabling compactions & flushes 2024-11-26T21:49:57,234 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1732657792023.651cf53d9624e6cc98e46d6578c71496. 2024-11-26T21:49:57,234 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1732657792023.651cf53d9624e6cc98e46d6578c71496. 2024-11-26T21:49:57,234 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1732657792023.651cf53d9624e6cc98e46d6578c71496. after waiting 0 ms 2024-11-26T21:49:57,234 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1732657792023.651cf53d9624e6cc98e46d6578c71496. 2024-11-26T21:49:57,239 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/651cf53d9624e6cc98e46d6578c71496/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T21:49:57,240 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1732657792023.651cf53d9624e6cc98e46d6578c71496. 2024-11-26T21:49:57,240 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 651cf53d9624e6cc98e46d6578c71496: Waiting for close lock at 1732657797234Running coprocessor pre-close hooks at 1732657797234Disabling compacts and flushes for region at 1732657797234Disabling writes for close at 1732657797234Writing region close event to WAL at 1732657797234Running coprocessor post-close hooks at 1732657797240 (+6 ms)Closed at 1732657797240 2024-11-26T21:49:57,240 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1732657792023.651cf53d9624e6cc98e46d6578c71496. 2024-11-26T21:49:57,240 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2b8b8290cc9d27b8d226e5dc2b7d770d, disabling compactions & flushes 2024-11-26T21:49:57,240 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1732657792023.2b8b8290cc9d27b8d226e5dc2b7d770d. 2024-11-26T21:49:57,240 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1732657792023.2b8b8290cc9d27b8d226e5dc2b7d770d. 2024-11-26T21:49:57,240 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1732657792023.2b8b8290cc9d27b8d226e5dc2b7d770d. after waiting 0 ms 2024-11-26T21:49:57,240 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1732657792023.2b8b8290cc9d27b8d226e5dc2b7d770d. 2024-11-26T21:49:57,245 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/data/default/test/2b8b8290cc9d27b8d226e5dc2b7d770d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T21:49:57,245 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1732657792023.2b8b8290cc9d27b8d226e5dc2b7d770d. 2024-11-26T21:49:57,246 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2b8b8290cc9d27b8d226e5dc2b7d770d: Waiting for close lock at 1732657797240Running coprocessor pre-close hooks at 1732657797240Disabling compacts and flushes for region at 1732657797240Disabling writes for close at 1732657797240Writing region close event to WAL at 1732657797241 (+1 ms)Running coprocessor post-close hooks at 1732657797245 (+4 ms)Closed at 1732657797245 2024-11-26T21:49:57,246 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1732657792023.2b8b8290cc9d27b8d226e5dc2b7d770d. 2024-11-26T21:49:57,318 INFO [RS:2;828b3f090c91:42273 {}] regionserver.HRegionServer(976): stopping server 828b3f090c91,42273,1732657789877; all regions closed. 2024-11-26T21:49:57,321 INFO [RS:1;828b3f090c91:40359 {}] regionserver.HRegionServer(976): stopping server 828b3f090c91,40359,1732657789839; all regions closed. 2024-11-26T21:49:57,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741835_1011 (size=7074) 2024-11-26T21:49:57,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741835_1011 (size=7074) 2024-11-26T21:49:57,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741835_1011 (size=7074) 2024-11-26T21:49:57,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741865_1041 (size=4192) 2024-11-26T21:49:57,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741865_1041 (size=4192) 2024-11-26T21:49:57,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741865_1041 (size=4192) 2024-11-26T21:49:57,328 DEBUG [RS:2;828b3f090c91:42273 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/oldWALs 2024-11-26T21:49:57,328 INFO [RS:2;828b3f090c91:42273 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 828b3f090c91%2C42273%2C1732657789877:(num 1732657791139) 2024-11-26T21:49:57,328 DEBUG [RS:2;828b3f090c91:42273 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T21:49:57,328 INFO [RS:2;828b3f090c91:42273 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T21:49:57,329 INFO [RS:2;828b3f090c91:42273 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T21:49:57,329 INFO [RS:2;828b3f090c91:42273 {}] hbase.ChoreService(370): Chore service for: regionserver/828b3f090c91:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-26T21:49:57,329 INFO [RS:2;828b3f090c91:42273 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T21:49:57,329 INFO [RS:2;828b3f090c91:42273 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T21:49:57,329 INFO [regionserver/828b3f090c91:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T21:49:57,329 INFO [RS:2;828b3f090c91:42273 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T21:49:57,329 INFO [RS:2;828b3f090c91:42273 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T21:49:57,330 INFO [RS:2;828b3f090c91:42273 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42273 2024-11-26T21:49:57,331 DEBUG [RS:1;828b3f090c91:40359 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/oldWALs 2024-11-26T21:49:57,331 INFO [RS:1;828b3f090c91:40359 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 828b3f090c91%2C40359%2C1732657789839.meta:.meta(num 1732657795083) 2024-11-26T21:49:57,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/828b3f090c91,42273,1732657789877 2024-11-26T21:49:57,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T21:49:57,331 INFO [RS:2;828b3f090c91:42273 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T21:49:57,332 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [828b3f090c91,42273,1732657789877] 2024-11-26T21:49:57,332 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/828b3f090c91,42273,1732657789877 already deleted, retry=false 2024-11-26T21:49:57,333 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 828b3f090c91,42273,1732657789877 expired; onlineServers=1 2024-11-26T21:49:57,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741834_1010 (size=5189) 2024-11-26T21:49:57,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741834_1010 (size=5189) 2024-11-26T21:49:57,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741834_1010 (size=5189) 2024-11-26T21:49:57,338 DEBUG [RS:1;828b3f090c91:40359 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/oldWALs 2024-11-26T21:49:57,338 INFO [RS:1;828b3f090c91:40359 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 828b3f090c91%2C40359%2C1732657789839:(num 1732657791102) 2024-11-26T21:49:57,338 DEBUG [RS:1;828b3f090c91:40359 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T21:49:57,338 INFO [RS:1;828b3f090c91:40359 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T21:49:57,338 INFO [RS:1;828b3f090c91:40359 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T21:49:57,338 INFO [RS:1;828b3f090c91:40359 {}] hbase.ChoreService(370): Chore service for: regionserver/828b3f090c91:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-26T21:49:57,339 INFO [RS:1;828b3f090c91:40359 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T21:49:57,339 INFO [regionserver/828b3f090c91:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T21:49:57,339 WARN [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.Threads(127): sleep interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2374) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T21:49:57,339 WARN [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,,1732657792023.bf200defbfcf83538e813f9fa91d9c35., aborting... java.io.IOException: Failed to report opened region to master: test,,1732657792023.bf200defbfcf83538e813f9fa91d9c35. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T21:49:57,339 ERROR [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2440): ***** ABORTING region server 828b3f090c91,40359,1732657789839: Failed to open region test,,1732657792023.bf200defbfcf83538e813f9fa91d9c35. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,,1732657792023.bf200defbfcf83538e813f9fa91d9c35. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T21:49:57,340 ERROR [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-26T21:49:57,340 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-26T21:49:57,340 INFO [RS:1;828b3f090c91:40359 {}] executor.ExecutorService(108): Executor-9-RS_OPEN_REGION-regionserver/828b3f090c91:0 had [Event #36 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler), Event #37 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler), Event #38 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler)] on shutdown 2024-11-26T21:49:57,341 INFO [RS:1;828b3f090c91:40359 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40359 2024-11-26T21:49:57,341 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-26T21:49:57,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/828b3f090c91,40359,1732657789839 2024-11-26T21:49:57,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T21:49:57,342 INFO [RS:1;828b3f090c91:40359 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T21:49:57,343 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [828b3f090c91,40359,1732657789839] 2024-11-26T21:49:57,344 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/828b3f090c91,40359,1732657789839 already deleted, retry=false 2024-11-26T21:49:57,344 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 828b3f090c91,40359,1732657789839 expired; onlineServers=0 2024-11-26T21:49:57,344 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-26T21:49:57,344 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '828b3f090c91,44203,1732657789166' ***** 2024-11-26T21:49:57,344 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-26T21:49:57,344 INFO [M:0;828b3f090c91:44203 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T21:49:57,344 INFO [M:0;828b3f090c91:44203 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T21:49:57,344 DEBUG [M:0;828b3f090c91:44203 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-26T21:49:57,345 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-26T21:49:57,345 DEBUG [M:0;828b3f090c91:44203 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-26T21:49:57,345 DEBUG [master/828b3f090c91:0:becomeActiveMaster-HFileCleaner.large.0-1732657790797 {}] cleaner.HFileCleaner(306): Exit Thread[master/828b3f090c91:0:becomeActiveMaster-HFileCleaner.large.0-1732657790797,5,FailOnTimeoutGroup] 2024-11-26T21:49:57,345 DEBUG [master/828b3f090c91:0:becomeActiveMaster-HFileCleaner.small.0-1732657790798 {}] cleaner.HFileCleaner(306): Exit Thread[master/828b3f090c91:0:becomeActiveMaster-HFileCleaner.small.0-1732657790798,5,FailOnTimeoutGroup] 2024-11-26T21:49:57,345 INFO [M:0;828b3f090c91:44203 {}] hbase.ChoreService(370): Chore service for: master/828b3f090c91:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-26T21:49:57,345 INFO [M:0;828b3f090c91:44203 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T21:49:57,345 DEBUG [M:0;828b3f090c91:44203 {}] master.HMaster(1795): Stopping service threads 2024-11-26T21:49:57,345 INFO [M:0;828b3f090c91:44203 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-26T21:49:57,345 INFO [M:0;828b3f090c91:44203 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:49:57,345 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-26T21:49:57,345 INFO [M:0;828b3f090c91:44203 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T21:49:57,346 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-26T21:49:57,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-26T21:49:57,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:49:57,346 DEBUG [M:0;828b3f090c91:44203 {}] zookeeper.ZKUtil(347): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-26T21:49:57,346 WARN [M:0;828b3f090c91:44203 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-26T21:49:57,347 INFO [M:0;828b3f090c91:44203 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/.lastflushedseqids 2024-11-26T21:49:57,357 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "Verbose": false, "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 474491136 }, "NonHeapMemoryUsage": { "committed": 162070528, "init": 7667712, "max": -1, "used": 159503624 }, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "828b3f090c91", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 40636, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 132, "ProcessCallTime_min": 0, "ProcessCallTime_max": 25, "ProcessCallTime_mean": 2, "ProcessCallTime_25th_percentile": 6, "ProcessCallTime_median": 12, "ProcessCallTime_75th_percentile": 18, "ProcessCallTime_90th_percentile": 22, "ProcessCallTime_95th_percentile": 23, "ProcessCallTime_98th_percentile": 24, "ProcessCallTime_99th_percentile": 24, "ProcessCallTime_99.9th_percentile": 24, "ProcessCallTime_TimeRangeCount_0-1": 132, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 132, "QueueCallTime_min": 0, "QueueCallTime_max": 13, "QueueCallTime_mean": 1, "QueueCallTime_25th_percentile": 3, "QueueCallTime_median": 6, "QueueCallTime_75th_percentile": 9, "QueueCallTime_90th_percentile": 11, "QueueCallTime_95th_percentile": 12, "QueueCallTime_98th_percentile": 12, "QueueCallTime_99th_percentile": 12, "QueueCallTime_99.9th_percentile": 12, "QueueCallTime_TimeRangeCount_0-1": 132, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 132, "TotalCallTime_min": 0, "TotalCallTime_max": 25, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 6, "TotalCallTime_median": 12, "TotalCallTime_75th_percentile": 18, "TotalCallTime_90th_percentile": 22, "TotalCallTime_95th_percentile": 23, "TotalCallTime_98th_percentile": 24, "TotalCallTime_99th_percentile": 24, "TotalCallTime_99.9th_percentile": 24, "TotalCallTime_TimeRangeCount_0-1": 132, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 132, "ResponseSize_min": 0, "ResponseSize_max": 22850, "ResponseSize_mean": 4499, "ResponseSize_25th_percentile": 5712, "ResponseSize_median": 11425, "ResponseSize_75th_percentile": 17137, "ResponseSize_90th_percentile": 20565, "ResponseSize_95th_percentile": 21707, "ResponseSize_98th_percentile": 22392, "ResponseSize_99th_percentile": 22621, "ResponseSize_99.9th_percentile": 22827, "ResponseSize_SizeRangeCount_0-10": 132, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 132, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 279, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 132, "sentBytes": 459889 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "828b3f090c91", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:53387", "tag.serverName": "828b3f090c91,46545,1732657789755", "tag.clusterId": "6d98cb23-728a-4b03-88ab-90833a5d474e", "tag.Context": "regionserver", "tag.Hostname": "828b3f090c91", "regionCount": 9, "storeCount": 12, "hlogFileCount": 2, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 28672, "memStoreHeapSize": 51520, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732657789755, "averageRegionSize": 3185, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 38191, "localBytesRead": 38191, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 18.8, "writeRequestRatePerSecond": 16.400000000000002, "ByteBuffAllocatorHeapAllocationBytes": 39379, "ByteBuffAllocatorPoolAllocationBytes": 465920, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 74, "totalRowActionRequestCount": 176, "readRequestCount": 94, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 82, "rpcGetRequestCount": 2, "rpcFullScanRequestCount": 8, "rpcScanRequestCount": 15, "rpcMultiRequestCount": 1, "rpcMutateRequestCount": 56, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 15, "ScanTime_min": 0, "ScanTime_max": 10, "ScanTime_mean": 1, "ScanTime_25th_percentile": 2, "ScanTime_median": 5, "ScanTime_75th_percentile": 7, "ScanTime_90th_percentile": 9, "ScanTime_95th_percentile": 9, "ScanTime_98th_percentile": 9, "ScanTime_99th_percentile": 9, "ScanTime_99.9th_percentile": 9, "ScanTime_TimeRangeCount_0-1": 15, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 56, "Put_min": 1, "Put_max": 6, "Put_mean": 2, "Put_25th_percentile": 2, "Put_median": 3, "Put_75th_percentile": 4, "Put_90th_percentile": 5, "Put_95th_percentile": 5, "Put_98th_percentile": 5, "Put_99th_percentile": 5, "Put_99.9th_percentile": 5, "Put_TimeRangeCount_0-1": 56, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 1, "PutBatch_min": 11, "PutBatch_max": 11, "PutBatch_mean": 11, "PutBatch_25th_percentile": 11, "PutBatch_median": 11, "PutBatch_75th_percentile": 11, "PutBatch_90th_percentile": 11, "PutBatch_95th_percentile": 11, "PutBatch_98th_percentile": 11, "PutBatch_99th_percentile": 11, "PutBatch_99.9th_percentile": 11, "PutBatch_TimeRangeCount_0-1": 1, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 2, "Get_min": 0, "Get_max": 2, "Get_mean": 1, "Get_25th_percentile": 0, "Get_median": 1, "Get_75th_percentile": 1, "Get_90th_percentile": 1, "Get_95th_percentile": 1, "Get_98th_percentile": 1, "Get_99th_percentile": 1, "Get_99.9th_percentile": 1, "Get_TimeRangeCount_0-1": 2, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 15, "ScanSize_min": 0, "ScanSize_max": 22232, "ScanSize_mean": 4465, "ScanSize_25th_percentile": 5558, "ScanSize_median": 11116, "ScanSize_75th_percentile": 16674, "ScanSize_90th_percentile": 20008, "ScanSize_95th_percentile": 21120, "ScanSize_98th_percentile": 21787, "ScanSize_99th_percentile": 22009, "ScanSize_99.9th_percentile": 22209, "ScanSize_SizeRangeCount_0-10": 15, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-26T21:49:57,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741870_1046 (size=1560) 2024-11-26T21:49:57,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741870_1046 (size=1560) 2024-11-26T21:49:57,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741870_1046 (size=1560) 2024-11-26T21:49:57,368 INFO [M:0;828b3f090c91:44203 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T21:49:57,368 INFO [M:0;828b3f090c91:44203 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-26T21:49:57,368 DEBUG [M:0;828b3f090c91:44203 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T21:49:57,368 INFO [M:0;828b3f090c91:44203 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T21:49:57,368 DEBUG [M:0;828b3f090c91:44203 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T21:49:57,368 DEBUG [M:0;828b3f090c91:44203 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T21:49:57,368 DEBUG [M:0;828b3f090c91:44203 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T21:49:57,368 INFO [M:0;828b3f090c91:44203 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=248.75 KB heapSize=302.30 KB 2024-11-26T21:49:57,385 DEBUG [M:0;828b3f090c91:44203 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/aa21ba6853bd44bd9ffe35b73995d93a is 82, key is hbase:meta,,1/info:regioninfo/1732657795842/Put/seqid=0 2024-11-26T21:49:57,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741871_1047 (size=6071) 2024-11-26T21:49:57,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741871_1047 (size=6071) 2024-11-26T21:49:57,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741871_1047 (size=6071) 2024-11-26T21:49:57,393 INFO [M:0;828b3f090c91:44203 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/aa21ba6853bd44bd9ffe35b73995d93a 2024-11-26T21:49:57,420 DEBUG [M:0;828b3f090c91:44203 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3e3b8e9372d54d659f283dc1d944d328 is 2987, key is \x00\x00\x00\x00\x00\x00\x009/proc:d/1732657797087/Put/seqid=0 2024-11-26T21:49:57,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741872_1048 (size=12799) 2024-11-26T21:49:57,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741872_1048 (size=12799) 2024-11-26T21:49:57,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741872_1048 (size=12799) 2024-11-26T21:49:57,429 INFO [M:0;828b3f090c91:44203 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=247.47 KB at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3e3b8e9372d54d659f283dc1d944d328 2024-11-26T21:49:57,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T21:49:57,433 INFO [RS:2;828b3f090c91:42273 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T21:49:57,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42273-0x1007e03121b0003, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T21:49:57,433 INFO [RS:2;828b3f090c91:42273 {}] regionserver.HRegionServer(1031): Exiting; stopping=828b3f090c91,42273,1732657789877; zookeeper connection closed. 2024-11-26T21:49:57,433 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@498550bc {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@498550bc 2024-11-26T21:49:57,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T21:49:57,443 INFO [RS:1;828b3f090c91:40359 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T21:49:57,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40359-0x1007e03121b0002, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T21:49:57,443 INFO [RS:1;828b3f090c91:40359 {}] regionserver.HRegionServer(1031): Exiting; stopping=828b3f090c91,40359,1732657789839; zookeeper connection closed. 2024-11-26T21:49:57,444 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@490df1e7 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@490df1e7 2024-11-26T21:49:57,444 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-26T21:49:57,454 DEBUG [M:0;828b3f090c91:44203 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a01c1ab329764870909319b412bb200a is 69, key is 828b3f090c91,40359,1732657789839/rs:state/1732657790842/Put/seqid=0 2024-11-26T21:49:57,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741873_1049 (size=5442) 2024-11-26T21:49:57,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741873_1049 (size=5442) 2024-11-26T21:49:57,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741873_1049 (size=5442) 2024-11-26T21:49:57,462 INFO [M:0;828b3f090c91:44203 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a01c1ab329764870909319b412bb200a 2024-11-26T21:49:57,468 INFO [M:0;828b3f090c91:44203 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a01c1ab329764870909319b412bb200a 2024-11-26T21:49:57,485 DEBUG [M:0;828b3f090c91:44203 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/8e1c1c6eeafd4fdbb3bb2fbed9673fe3 is 52, key is load_balancer_on/state:d/1732657794373/Put/seqid=0 2024-11-26T21:49:57,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741874_1050 (size=5058) 2024-11-26T21:49:57,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741874_1050 (size=5058) 2024-11-26T21:49:57,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741874_1050 (size=5058) 2024-11-26T21:49:57,499 INFO [M:0;828b3f090c91:44203 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/8e1c1c6eeafd4fdbb3bb2fbed9673fe3 2024-11-26T21:49:57,508 DEBUG [M:0;828b3f090c91:44203 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/aa21ba6853bd44bd9ffe35b73995d93a as hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/aa21ba6853bd44bd9ffe35b73995d93a 2024-11-26T21:49:57,514 INFO [M:0;828b3f090c91:44203 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/aa21ba6853bd44bd9ffe35b73995d93a, entries=14, sequenceid=571, filesize=5.9 K 2024-11-26T21:49:57,515 DEBUG [M:0;828b3f090c91:44203 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3e3b8e9372d54d659f283dc1d944d328 as hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3e3b8e9372d54d659f283dc1d944d328 2024-11-26T21:49:57,522 INFO [M:0;828b3f090c91:44203 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3e3b8e9372d54d659f283dc1d944d328, entries=79, sequenceid=571, filesize=12.5 K 2024-11-26T21:49:57,523 DEBUG [M:0;828b3f090c91:44203 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a01c1ab329764870909319b412bb200a as hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a01c1ab329764870909319b412bb200a 2024-11-26T21:49:57,530 INFO [M:0;828b3f090c91:44203 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a01c1ab329764870909319b412bb200a 2024-11-26T21:49:57,530 INFO [M:0;828b3f090c91:44203 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a01c1ab329764870909319b412bb200a, entries=3, sequenceid=571, filesize=5.3 K 2024-11-26T21:49:57,531 DEBUG [M:0;828b3f090c91:44203 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/8e1c1c6eeafd4fdbb3bb2fbed9673fe3 as hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/8e1c1c6eeafd4fdbb3bb2fbed9673fe3 2024-11-26T21:49:57,538 INFO [M:0;828b3f090c91:44203 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45749/user/jenkins/test-data/faed6e06-d11c-b89e-32b9-73b4deabb657/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/8e1c1c6eeafd4fdbb3bb2fbed9673fe3, entries=1, sequenceid=571, filesize=4.9 K 2024-11-26T21:49:57,539 INFO [M:0;828b3f090c91:44203 {}] regionserver.HRegion(3140): Finished flush of dataSize ~248.75 KB/254715, heapSize ~302.23 KB/309488, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 171ms, sequenceid=571, compaction requested=false 2024-11-26T21:49:57,541 INFO [M:0;828b3f090c91:44203 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T21:49:57,541 DEBUG [M:0;828b3f090c91:44203 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732657797368Disabling compacts and flushes for region at 1732657797368Disabling writes for close at 1732657797368Obtaining lock to block concurrent updates at 1732657797368Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732657797368Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=254715, getHeapSize=309488, getOffHeapSize=0, getCellsCount=710 at 1732657797369 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732657797369Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732657797369Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732657797385 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732657797385Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732657797399 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732657797420 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732657797420Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732657797435 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732657797453 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732657797453Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732657797468 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732657797484 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732657797484Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f033ffa: reopening flushed file at 1732657797506 (+22 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4900a37b: reopening flushed file at 1732657797514 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5dcbd433: reopening flushed file at 1732657797522 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d2d3fb2: reopening flushed file at 1732657797530 (+8 ms)Finished flush of dataSize ~248.75 KB/254715, heapSize ~302.23 KB/309488, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 171ms, sequenceid=571, compaction requested=false at 1732657797539 (+9 ms)Writing region close event to WAL at 1732657797541 (+2 ms)Closed at 1732657797541 2024-11-26T21:49:57,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40345 is added to blk_1073741830_1006 (size=292839) 2024-11-26T21:49:57,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39537 is added to blk_1073741830_1006 (size=292839) 2024-11-26T21:49:57,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43659 is added to blk_1073741830_1006 (size=292839) 2024-11-26T21:49:57,545 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T21:49:57,545 INFO [M:0;828b3f090c91:44203 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-26T21:49:57,545 INFO [M:0;828b3f090c91:44203 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44203 2024-11-26T21:49:57,545 INFO [M:0;828b3f090c91:44203 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T21:49:57,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T21:49:57,647 INFO [M:0;828b3f090c91:44203 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T21:49:57,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44203-0x1007e03121b0000, quorum=127.0.0.1:53387, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T21:49:57,656 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@67ddfd25{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T21:49:57,660 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@30e38263{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T21:49:57,660 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T21:49:57,661 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@52cf89e6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T21:49:57,661 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@11ebaa12{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/hadoop.log.dir/,STOPPED} 2024-11-26T21:49:57,664 WARN [BP-655727968-172.17.0.2-1732657786342 heartbeating to localhost/127.0.0.1:45749 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T21:49:57,664 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T21:49:57,664 WARN [BP-655727968-172.17.0.2-1732657786342 heartbeating to localhost/127.0.0.1:45749 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-655727968-172.17.0.2-1732657786342 (Datanode Uuid 72e127b1-5812-444a-9dfb-62013621bbf8) service to localhost/127.0.0.1:45749 2024-11-26T21:49:57,664 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T21:49:57,666 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/cluster_626ec74d-0f86-b49f-b08d-4173446b08f3/data/data5/current/BP-655727968-172.17.0.2-1732657786342 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T21:49:57,666 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/cluster_626ec74d-0f86-b49f-b08d-4173446b08f3/data/data6/current/BP-655727968-172.17.0.2-1732657786342 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T21:49:57,666 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T21:49:57,670 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7cfccf27{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T21:49:57,671 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6c41bbb7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T21:49:57,671 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T21:49:57,671 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@12f16529{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T21:49:57,671 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@34a3e1ff{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/hadoop.log.dir/,STOPPED} 2024-11-26T21:49:57,672 WARN [BP-655727968-172.17.0.2-1732657786342 heartbeating to localhost/127.0.0.1:45749 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T21:49:57,672 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T21:49:57,672 WARN [BP-655727968-172.17.0.2-1732657786342 heartbeating to localhost/127.0.0.1:45749 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-655727968-172.17.0.2-1732657786342 (Datanode Uuid b26d6d41-1899-4034-82ea-25ab341b922e) service to localhost/127.0.0.1:45749 2024-11-26T21:49:57,672 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T21:49:57,673 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/cluster_626ec74d-0f86-b49f-b08d-4173446b08f3/data/data3/current/BP-655727968-172.17.0.2-1732657786342 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T21:49:57,673 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/cluster_626ec74d-0f86-b49f-b08d-4173446b08f3/data/data4/current/BP-655727968-172.17.0.2-1732657786342 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T21:49:57,673 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T21:49:57,675 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2226d6a6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T21:49:57,675 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1da1f773{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T21:49:57,675 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T21:49:57,676 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1c5d2af1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T21:49:57,676 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@739a104e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/hadoop.log.dir/,STOPPED} 2024-11-26T21:49:57,677 WARN [BP-655727968-172.17.0.2-1732657786342 heartbeating to localhost/127.0.0.1:45749 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T21:49:57,677 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T21:49:57,677 WARN [BP-655727968-172.17.0.2-1732657786342 heartbeating to localhost/127.0.0.1:45749 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-655727968-172.17.0.2-1732657786342 (Datanode Uuid 6fd10630-2eb6-418e-a56d-198777b09dd3) service to localhost/127.0.0.1:45749 2024-11-26T21:49:57,677 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T21:49:57,678 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/cluster_626ec74d-0f86-b49f-b08d-4173446b08f3/data/data1/current/BP-655727968-172.17.0.2-1732657786342 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T21:49:57,678 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/cluster_626ec74d-0f86-b49f-b08d-4173446b08f3/data/data2/current/BP-655727968-172.17.0.2-1732657786342 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T21:49:57,678 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T21:49:57,684 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@376fa8f3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T21:49:57,685 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@672b88ea{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T21:49:57,685 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T21:49:57,685 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@76beb053{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T21:49:57,685 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@265b24f1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7b373903-0a6e-645f-77b2-9979101158e5/hadoop.log.dir/,STOPPED} 2024-11-26T21:49:57,699 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-26T21:49:57,757 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down