2024-11-17 02:03:46,583 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-17 02:03:46,600 main DEBUG Took 0.014670 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-17 02:03:46,600 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-17 02:03:46,600 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-17 02:03:46,601 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-17 02:03:46,602 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 02:03:46,608 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-17 02:03:46,619 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 02:03:46,620 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 02:03:46,620 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 02:03:46,621 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 02:03:46,621 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 02:03:46,621 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 02:03:46,622 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 02:03:46,623 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 02:03:46,623 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 02:03:46,623 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 02:03:46,624 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 02:03:46,625 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 02:03:46,625 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 02:03:46,625 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 02:03:46,626 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 02:03:46,626 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 02:03:46,627 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 02:03:46,627 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 02:03:46,627 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 02:03:46,628 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 02:03:46,628 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 02:03:46,628 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 02:03:46,629 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 02:03:46,629 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 02:03:46,629 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 02:03:46,630 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-17 02:03:46,631 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 02:03:46,632 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-17 02:03:46,634 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-17 02:03:46,634 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-17 02:03:46,635 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-17 02:03:46,636 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-17 02:03:46,643 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-17 02:03:46,645 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-17 02:03:46,647 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-17 02:03:46,647 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-17 02:03:46,648 main DEBUG createAppenders(={Console}) 2024-11-17 02:03:46,648 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-11-17 02:03:46,649 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-17 02:03:46,649 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-11-17 02:03:46,649 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-17 02:03:46,650 main DEBUG OutputStream closed 2024-11-17 02:03:46,650 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-17 02:03:46,650 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-17 02:03:46,650 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-11-17 02:03:46,716 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-17 02:03:46,718 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-17 02:03:46,719 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-17 02:03:46,719 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-17 02:03:46,720 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-17 02:03:46,720 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-17 02:03:46,720 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-17 02:03:46,721 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-17 02:03:46,721 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-17 02:03:46,721 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-17 02:03:46,722 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-17 02:03:46,722 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-17 02:03:46,722 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-17 02:03:46,723 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-17 02:03:46,723 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-17 02:03:46,723 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-17 02:03:46,723 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-17 02:03:46,724 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-17 02:03:46,726 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-17 02:03:46,726 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-11-17 02:03:46,727 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-17 02:03:46,727 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-11-17T02:03:46,739 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-11-17 02:03:46,742 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-17 02:03:46,742 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-17T02:03:46,957 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846 2024-11-17T02:03:46,977 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-17T02:03:46,997 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/cluster_e712193e-aef2-a358-3728-f7558b6dc143, deleteOnExit=true 2024-11-17T02:03:46,997 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-17T02:03:46,999 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/test.cache.data in system properties and HBase conf 2024-11-17T02:03:47,000 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/hadoop.tmp.dir in system properties and HBase conf 2024-11-17T02:03:47,001 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/hadoop.log.dir in system properties and HBase conf 2024-11-17T02:03:47,001 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-17T02:03:47,002 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-17T02:03:47,003 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-17T02:03:47,088 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-17T02:03:47,175 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T02:03:47,179 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-17T02:03:47,180 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-17T02:03:47,180 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-17T02:03:47,181 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-17T02:03:47,181 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-17T02:03:47,182 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-17T02:03:47,182 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-17T02:03:47,183 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-17T02:03:47,183 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-17T02:03:47,183 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/nfs.dump.dir in system properties and HBase conf 2024-11-17T02:03:47,184 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/java.io.tmpdir in system properties and HBase conf 2024-11-17T02:03:47,184 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-17T02:03:47,184 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-17T02:03:47,185 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-17T02:03:48,214 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-17T02:03:48,283 INFO [Time-limited test {}] log.Log(170): Logging initialized @2348ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-17T02:03:48,351 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T02:03:48,412 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-17T02:03:48,436 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-17T02:03:48,436 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-17T02:03:48,437 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-17T02:03:48,449 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T02:03:48,452 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/hadoop.log.dir/,AVAILABLE} 2024-11-17T02:03:48,453 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-17T02:03:48,635 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64418798{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/java.io.tmpdir/jetty-localhost-33517-hadoop-hdfs-3_4_1-tests_jar-_-any-5924893372470807239/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-17T02:03:48,641 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:33517} 2024-11-17T02:03:48,641 INFO [Time-limited test {}] server.Server(415): Started @2707ms 2024-11-17T02:03:49,108 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T02:03:49,116 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-17T02:03:49,117 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-17T02:03:49,117 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-17T02:03:49,117 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-17T02:03:49,118 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3a300759{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/hadoop.log.dir/,AVAILABLE} 2024-11-17T02:03:49,119 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@474d518f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-17T02:03:49,218 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@79b30fb0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/java.io.tmpdir/jetty-localhost-36525-hadoop-hdfs-3_4_1-tests_jar-_-any-3004276412185029202/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-17T02:03:49,219 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@548cd725{HTTP/1.1, (http/1.1)}{localhost:36525} 2024-11-17T02:03:49,219 INFO [Time-limited test {}] server.Server(415): Started @3285ms 2024-11-17T02:03:49,267 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-17T02:03:49,371 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T02:03:49,378 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-17T02:03:49,379 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-17T02:03:49,380 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-17T02:03:49,380 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-17T02:03:49,382 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@29652cf4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/hadoop.log.dir/,AVAILABLE} 2024-11-17T02:03:49,383 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1b0a32ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-17T02:03:49,483 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@34fcb585{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/java.io.tmpdir/jetty-localhost-33509-hadoop-hdfs-3_4_1-tests_jar-_-any-8082205615281386638/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-17T02:03:49,484 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@389bb6f7{HTTP/1.1, (http/1.1)}{localhost:33509} 2024-11-17T02:03:49,484 INFO [Time-limited test {}] server.Server(415): Started @3550ms 2024-11-17T02:03:49,486 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-17T02:03:49,526 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T02:03:49,533 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-17T02:03:49,535 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-17T02:03:49,535 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-17T02:03:49,536 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-17T02:03:49,537 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@290e2eec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/hadoop.log.dir/,AVAILABLE} 2024-11-17T02:03:49,538 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@417efacb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-17T02:03:49,656 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3c1f701{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/java.io.tmpdir/jetty-localhost-43625-hadoop-hdfs-3_4_1-tests_jar-_-any-69266802933182324/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-17T02:03:49,657 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3d599bb4{HTTP/1.1, (http/1.1)}{localhost:43625} 2024-11-17T02:03:49,657 INFO [Time-limited test {}] server.Server(415): Started @3723ms 2024-11-17T02:03:49,659 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-17T02:03:50,567 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/cluster_e712193e-aef2-a358-3728-f7558b6dc143/data/data4/current/BP-927759803-172.17.0.2-1731809027758/current, will proceed with Du for space computation calculation, 2024-11-17T02:03:50,567 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/cluster_e712193e-aef2-a358-3728-f7558b6dc143/data/data2/current/BP-927759803-172.17.0.2-1731809027758/current, will proceed with Du for space computation calculation, 2024-11-17T02:03:50,567 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/cluster_e712193e-aef2-a358-3728-f7558b6dc143/data/data1/current/BP-927759803-172.17.0.2-1731809027758/current, will proceed with Du for space computation calculation, 2024-11-17T02:03:50,567 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/cluster_e712193e-aef2-a358-3728-f7558b6dc143/data/data3/current/BP-927759803-172.17.0.2-1731809027758/current, will proceed with Du for space computation calculation, 2024-11-17T02:03:50,611 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-17T02:03:50,611 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-17T02:03:50,623 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/cluster_e712193e-aef2-a358-3728-f7558b6dc143/data/data5/current/BP-927759803-172.17.0.2-1731809027758/current, will proceed with Du for space computation calculation, 2024-11-17T02:03:50,623 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/cluster_e712193e-aef2-a358-3728-f7558b6dc143/data/data6/current/BP-927759803-172.17.0.2-1731809027758/current, will proceed with Du for space computation calculation, 2024-11-17T02:03:50,651 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-17T02:03:50,665 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x583f425319fa6ecf with lease ID 0x18b2b899c6ce2af4: Processing first storage report for DS-da656d4a-0400-49ad-97b4-848c04dabe50 from datanode DatanodeRegistration(127.0.0.1:36455, datanodeUuid=4107a711-3cdd-4db3-a24e-f4f492515acf, infoPort=37075, infoSecurePort=0, ipcPort=46329, storageInfo=lv=-57;cid=testClusterID;nsid=190144626;c=1731809027758) 2024-11-17T02:03:50,666 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x583f425319fa6ecf with lease ID 0x18b2b899c6ce2af4: from storage DS-da656d4a-0400-49ad-97b4-848c04dabe50 node DatanodeRegistration(127.0.0.1:36455, datanodeUuid=4107a711-3cdd-4db3-a24e-f4f492515acf, infoPort=37075, infoSecurePort=0, ipcPort=46329, storageInfo=lv=-57;cid=testClusterID;nsid=190144626;c=1731809027758), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-17T02:03:50,667 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846 2024-11-17T02:03:50,667 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe52e6024201fd3d with lease ID 0x18b2b899c6ce2af3: Processing first storage report for DS-19b795a5-ad52-4253-ba85-390185288d22 from datanode DatanodeRegistration(127.0.0.1:33655, datanodeUuid=92058095-02df-4731-a81d-1580a9bbc0f1, infoPort=40369, infoSecurePort=0, ipcPort=38605, storageInfo=lv=-57;cid=testClusterID;nsid=190144626;c=1731809027758) 2024-11-17T02:03:50,667 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe52e6024201fd3d with lease ID 0x18b2b899c6ce2af3: from storage DS-19b795a5-ad52-4253-ba85-390185288d22 node DatanodeRegistration(127.0.0.1:33655, datanodeUuid=92058095-02df-4731-a81d-1580a9bbc0f1, infoPort=40369, infoSecurePort=0, ipcPort=38605, storageInfo=lv=-57;cid=testClusterID;nsid=190144626;c=1731809027758), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-17T02:03:50,668 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd146426997e24da1 with lease ID 0x18b2b899c6ce2af2: Processing first storage report for DS-11c5e779-9b67-42ae-934a-e223a96170e4 from datanode DatanodeRegistration(127.0.0.1:36067, datanodeUuid=b4ea61d2-621c-4f78-9946-a6c292a6b2a5, infoPort=40831, infoSecurePort=0, ipcPort=36791, storageInfo=lv=-57;cid=testClusterID;nsid=190144626;c=1731809027758) 2024-11-17T02:03:50,668 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd146426997e24da1 with lease ID 0x18b2b899c6ce2af2: from storage DS-11c5e779-9b67-42ae-934a-e223a96170e4 node DatanodeRegistration(127.0.0.1:36067, datanodeUuid=b4ea61d2-621c-4f78-9946-a6c292a6b2a5, infoPort=40831, infoSecurePort=0, ipcPort=36791, storageInfo=lv=-57;cid=testClusterID;nsid=190144626;c=1731809027758), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-17T02:03:50,668 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x583f425319fa6ecf with lease ID 0x18b2b899c6ce2af4: Processing first storage report for DS-8343c4fa-abf8-41db-ae34-356a95f7f3f6 from datanode DatanodeRegistration(127.0.0.1:36455, datanodeUuid=4107a711-3cdd-4db3-a24e-f4f492515acf, infoPort=37075, infoSecurePort=0, ipcPort=46329, storageInfo=lv=-57;cid=testClusterID;nsid=190144626;c=1731809027758) 2024-11-17T02:03:50,669 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x583f425319fa6ecf with lease ID 0x18b2b899c6ce2af4: from storage DS-8343c4fa-abf8-41db-ae34-356a95f7f3f6 node DatanodeRegistration(127.0.0.1:36455, datanodeUuid=4107a711-3cdd-4db3-a24e-f4f492515acf, infoPort=37075, infoSecurePort=0, ipcPort=46329, storageInfo=lv=-57;cid=testClusterID;nsid=190144626;c=1731809027758), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-17T02:03:50,669 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe52e6024201fd3d with lease ID 0x18b2b899c6ce2af3: Processing first storage report for DS-af811735-5269-4824-ac6e-8dac0e0b90b3 from datanode DatanodeRegistration(127.0.0.1:33655, datanodeUuid=92058095-02df-4731-a81d-1580a9bbc0f1, infoPort=40369, infoSecurePort=0, ipcPort=38605, storageInfo=lv=-57;cid=testClusterID;nsid=190144626;c=1731809027758) 2024-11-17T02:03:50,669 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe52e6024201fd3d with lease ID 0x18b2b899c6ce2af3: from storage DS-af811735-5269-4824-ac6e-8dac0e0b90b3 node DatanodeRegistration(127.0.0.1:33655, datanodeUuid=92058095-02df-4731-a81d-1580a9bbc0f1, infoPort=40369, infoSecurePort=0, ipcPort=38605, storageInfo=lv=-57;cid=testClusterID;nsid=190144626;c=1731809027758), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-17T02:03:50,670 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd146426997e24da1 with lease ID 0x18b2b899c6ce2af2: Processing first storage report for DS-c5b64596-3c4a-430f-bc26-321b081bc72c from datanode DatanodeRegistration(127.0.0.1:36067, datanodeUuid=b4ea61d2-621c-4f78-9946-a6c292a6b2a5, infoPort=40831, infoSecurePort=0, ipcPort=36791, storageInfo=lv=-57;cid=testClusterID;nsid=190144626;c=1731809027758) 2024-11-17T02:03:50,670 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd146426997e24da1 with lease ID 0x18b2b899c6ce2af2: from storage DS-c5b64596-3c4a-430f-bc26-321b081bc72c node DatanodeRegistration(127.0.0.1:36067, datanodeUuid=b4ea61d2-621c-4f78-9946-a6c292a6b2a5, infoPort=40831, infoSecurePort=0, ipcPort=36791, storageInfo=lv=-57;cid=testClusterID;nsid=190144626;c=1731809027758), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-17T02:03:50,735 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/cluster_e712193e-aef2-a358-3728-f7558b6dc143/zookeeper_0, clientPort=53449, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/cluster_e712193e-aef2-a358-3728-f7558b6dc143/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/cluster_e712193e-aef2-a358-3728-f7558b6dc143/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-17T02:03:50,745 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53449 2024-11-17T02:03:50,761 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T02:03:50,766 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T02:03:50,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741825_1001 (size=7) 2024-11-17T02:03:50,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741825_1001 (size=7) 2024-11-17T02:03:50,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741825_1001 (size=7) 2024-11-17T02:03:51,384 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe with version=8 2024-11-17T02:03:51,385 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/hbase-staging 2024-11-17T02:03:51,458 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-17T02:03:51,676 INFO [Time-limited test {}] client.ConnectionUtils(128): master/c6c0d7a0a7c0:0 server-side Connection retries=45 2024-11-17T02:03:51,684 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T02:03:51,684 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-17T02:03:51,688 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-17T02:03:51,689 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T02:03:51,689 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-17T02:03:51,815 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-17T02:03:51,868 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-17T02:03:51,877 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-17T02:03:51,880 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-17T02:03:51,903 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 103440 (auto-detected) 2024-11-17T02:03:51,904 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-17T02:03:51,923 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:32803 2024-11-17T02:03:51,949 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:32803 connecting to ZooKeeper ensemble=127.0.0.1:53449 2024-11-17T02:03:52,061 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:328030x0, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-17T02:03:52,065 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:32803-0x10147d245010000 connected 2024-11-17T02:03:52,148 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T02:03:52,150 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T02:03:52,160 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T02:03:52,164 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe, hbase.cluster.distributed=false 2024-11-17T02:03:52,185 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-17T02:03:52,191 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=32803 2024-11-17T02:03:52,191 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=32803 2024-11-17T02:03:52,192 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=32803 2024-11-17T02:03:52,192 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=32803 2024-11-17T02:03:52,193 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=32803 2024-11-17T02:03:52,295 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c6c0d7a0a7c0:0 server-side Connection retries=45 2024-11-17T02:03:52,296 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T02:03:52,297 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-17T02:03:52,297 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-17T02:03:52,297 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T02:03:52,297 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-17T02:03:52,300 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-17T02:03:52,303 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-17T02:03:52,304 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33127 2024-11-17T02:03:52,306 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33127 connecting to ZooKeeper ensemble=127.0.0.1:53449 2024-11-17T02:03:52,307 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T02:03:52,312 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T02:03:52,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:331270x0, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-17T02:03:52,353 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:331270x0, quorum=127.0.0.1:53449, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T02:03:52,353 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33127-0x10147d245010001 connected 2024-11-17T02:03:52,358 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-17T02:03:52,368 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-17T02:03:52,371 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-17T02:03:52,376 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-17T02:03:52,377 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33127 2024-11-17T02:03:52,377 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33127 2024-11-17T02:03:52,378 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33127 2024-11-17T02:03:52,378 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33127 2024-11-17T02:03:52,378 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33127 2024-11-17T02:03:52,395 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c6c0d7a0a7c0:0 server-side Connection retries=45 2024-11-17T02:03:52,395 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T02:03:52,395 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-17T02:03:52,396 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-17T02:03:52,396 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T02:03:52,396 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-17T02:03:52,396 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-17T02:03:52,397 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-17T02:03:52,398 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45037 2024-11-17T02:03:52,400 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45037 connecting to ZooKeeper ensemble=127.0.0.1:53449 2024-11-17T02:03:52,401 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T02:03:52,406 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T02:03:52,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:450370x0, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-17T02:03:52,430 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45037-0x10147d245010002 connected 2024-11-17T02:03:52,430 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:450370x0, quorum=127.0.0.1:53449, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T02:03:52,431 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-17T02:03:52,432 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-17T02:03:52,433 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-17T02:03:52,435 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-17T02:03:52,435 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45037 2024-11-17T02:03:52,436 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45037 2024-11-17T02:03:52,436 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45037 2024-11-17T02:03:52,436 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45037 2024-11-17T02:03:52,437 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45037 2024-11-17T02:03:52,456 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c6c0d7a0a7c0:0 server-side Connection retries=45 2024-11-17T02:03:52,456 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T02:03:52,456 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-17T02:03:52,457 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-17T02:03:52,457 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T02:03:52,457 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-17T02:03:52,457 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-17T02:03:52,458 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-17T02:03:52,459 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34033 2024-11-17T02:03:52,461 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34033 connecting to ZooKeeper ensemble=127.0.0.1:53449 2024-11-17T02:03:52,463 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T02:03:52,466 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T02:03:52,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:340330x0, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-17T02:03:52,503 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:340330x0, quorum=127.0.0.1:53449, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T02:03:52,503 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34033-0x10147d245010003 connected 2024-11-17T02:03:52,504 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-17T02:03:52,505 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-17T02:03:52,506 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34033-0x10147d245010003, quorum=127.0.0.1:53449, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-17T02:03:52,507 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34033-0x10147d245010003, quorum=127.0.0.1:53449, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-17T02:03:52,508 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34033 2024-11-17T02:03:52,508 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34033 2024-11-17T02:03:52,509 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34033 2024-11-17T02:03:52,515 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34033 2024-11-17T02:03:52,515 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34033 2024-11-17T02:03:52,529 DEBUG [M:0;c6c0d7a0a7c0:32803 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c6c0d7a0a7c0:32803 2024-11-17T02:03:52,531 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c6c0d7a0a7c0,32803,1731809031530 2024-11-17T02:03:52,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T02:03:52,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34033-0x10147d245010003, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T02:03:52,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T02:03:52,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T02:03:52,558 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c6c0d7a0a7c0,32803,1731809031530 2024-11-17T02:03:52,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-17T02:03:52,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34033-0x10147d245010003, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-17T02:03:52,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T02:03:52,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-17T02:03:52,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34033-0x10147d245010003, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T02:03:52,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T02:03:52,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T02:03:52,588 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-17T02:03:52,589 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c6c0d7a0a7c0,32803,1731809031530 from backup master directory 2024-11-17T02:03:52,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34033-0x10147d245010003, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T02:03:52,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c6c0d7a0a7c0,32803,1731809031530 2024-11-17T02:03:52,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T02:03:52,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T02:03:52,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T02:03:52,603 WARN [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-17T02:03:52,603 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c6c0d7a0a7c0,32803,1731809031530 2024-11-17T02:03:52,606 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-17T02:03:52,607 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-17T02:03:52,662 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/hbase.id] with ID: 4e2710dd-e280-4e22-bcbe-5222a4ddb079 2024-11-17T02:03:52,663 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/.tmp/hbase.id 2024-11-17T02:03:52,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741826_1002 (size=42) 2024-11-17T02:03:52,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741826_1002 (size=42) 2024-11-17T02:03:52,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741826_1002 (size=42) 2024-11-17T02:03:52,679 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/.tmp/hbase.id]:[hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/hbase.id] 2024-11-17T02:03:52,724 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T02:03:52,730 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-17T02:03:52,749 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-17T02:03:52,779 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T02:03:52,779 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T02:03:52,779 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34033-0x10147d245010003, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T02:03:52,779 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T02:03:52,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741827_1003 (size=196) 2024-11-17T02:03:52,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741827_1003 (size=196) 2024-11-17T02:03:52,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741827_1003 (size=196) 2024-11-17T02:03:52,819 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T02:03:52,821 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-17T02:03:52,834 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T02:03:52,837 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T02:03:52,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741828_1004 (size=1189) 2024-11-17T02:03:52,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741828_1004 (size=1189) 2024-11-17T02:03:52,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741828_1004 (size=1189) 2024-11-17T02:03:52,886 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/MasterData/data/master/store 2024-11-17T02:03:52,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741829_1005 (size=34) 2024-11-17T02:03:52,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741829_1005 (size=34) 2024-11-17T02:03:52,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741829_1005 (size=34) 2024-11-17T02:03:52,913 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-17T02:03:52,916 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:52,917 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-17T02:03:52,918 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T02:03:52,918 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T02:03:52,920 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-17T02:03:52,921 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T02:03:52,921 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T02:03:52,923 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731809032917Disabling compacts and flushes for region at 1731809032917Disabling writes for close at 1731809032921 (+4 ms)Writing region close event to WAL at 1731809032921Closed at 1731809032921 2024-11-17T02:03:52,926 WARN [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/MasterData/data/master/store/.initializing 2024-11-17T02:03:52,926 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/MasterData/WALs/c6c0d7a0a7c0,32803,1731809031530 2024-11-17T02:03:52,936 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T02:03:52,951 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c6c0d7a0a7c0%2C32803%2C1731809031530, suffix=, logDir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/MasterData/WALs/c6c0d7a0a7c0,32803,1731809031530, archiveDir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/MasterData/oldWALs, maxLogs=10 2024-11-17T02:03:52,975 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/MasterData/WALs/c6c0d7a0a7c0,32803,1731809031530/c6c0d7a0a7c0%2C32803%2C1731809031530.1731809032955, exclude list is [], retry=0 2024-11-17T02:03:52,994 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36455,DS-da656d4a-0400-49ad-97b4-848c04dabe50,DISK] 2024-11-17T02:03:52,994 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36067,DS-11c5e779-9b67-42ae-934a-e223a96170e4,DISK] 2024-11-17T02:03:52,994 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33655,DS-19b795a5-ad52-4253-ba85-390185288d22,DISK] 2024-11-17T02:03:52,997 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-17T02:03:53,040 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/MasterData/WALs/c6c0d7a0a7c0,32803,1731809031530/c6c0d7a0a7c0%2C32803%2C1731809031530.1731809032955 2024-11-17T02:03:53,040 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40369:40369),(127.0.0.1/127.0.0.1:37075:37075),(127.0.0.1/127.0.0.1:40831:40831)] 2024-11-17T02:03:53,041 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-17T02:03:53,041 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:53,044 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T02:03:53,045 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T02:03:53,081 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T02:03:53,104 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-17T02:03:53,107 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:53,110 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T02:03:53,110 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T02:03:53,114 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-17T02:03:53,114 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:53,115 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T02:03:53,115 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T02:03:53,119 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-17T02:03:53,120 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:53,121 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T02:03:53,121 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T02:03:53,124 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-17T02:03:53,124 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:53,125 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T02:03:53,126 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T02:03:53,129 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-17T02:03:53,131 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-17T02:03:53,136 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T02:03:53,137 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T02:03:53,141 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-17T02:03:53,144 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T02:03:53,149 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T02:03:53,150 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61662337, jitterRate=-0.0811595767736435}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-17T02:03:53,158 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731809033057Initializing all the Stores at 1731809033059 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731809033059Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731809033060 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731809033060Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731809033061 (+1 ms)Cleaning up temporary data from old regions at 1731809033137 (+76 ms)Region opened successfully at 1731809033158 (+21 ms) 2024-11-17T02:03:53,159 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-17T02:03:53,189 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@155fed1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c6c0d7a0a7c0/172.17.0.2:0 2024-11-17T02:03:53,215 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-17T02:03:53,225 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-17T02:03:53,225 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-17T02:03:53,228 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-17T02:03:53,229 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-17T02:03:53,233 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-17T02:03:53,233 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-17T02:03:53,257 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-17T02:03:53,265 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-17T02:03:53,321 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-17T02:03:53,324 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-17T02:03:53,327 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-17T02:03:53,337 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-17T02:03:53,340 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-17T02:03:53,343 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-17T02:03:53,354 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-17T02:03:53,356 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-17T02:03:53,362 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-17T02:03:53,381 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-17T02:03:53,387 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-17T02:03:53,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34033-0x10147d245010003, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-17T02:03:53,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-17T02:03:53,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-17T02:03:53,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-17T02:03:53,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34033-0x10147d245010003, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T02:03:53,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T02:03:53,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T02:03:53,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T02:03:53,399 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c6c0d7a0a7c0,32803,1731809031530, sessionid=0x10147d245010000, setting cluster-up flag (Was=false) 2024-11-17T02:03:53,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T02:03:53,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34033-0x10147d245010003, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T02:03:53,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T02:03:53,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T02:03:53,454 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-17T02:03:53,458 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c6c0d7a0a7c0,32803,1731809031530 2024-11-17T02:03:53,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T02:03:53,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T02:03:53,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34033-0x10147d245010003, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T02:03:53,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T02:03:53,504 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-17T02:03:53,506 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c6c0d7a0a7c0,32803,1731809031530 2024-11-17T02:03:53,513 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-17T02:03:53,519 INFO [RS:2;c6c0d7a0a7c0:34033 {}] regionserver.HRegionServer(746): ClusterId : 4e2710dd-e280-4e22-bcbe-5222a4ddb079 2024-11-17T02:03:53,519 INFO [RS:0;c6c0d7a0a7c0:33127 {}] regionserver.HRegionServer(746): ClusterId : 4e2710dd-e280-4e22-bcbe-5222a4ddb079 2024-11-17T02:03:53,519 INFO [RS:1;c6c0d7a0a7c0:45037 {}] regionserver.HRegionServer(746): ClusterId : 4e2710dd-e280-4e22-bcbe-5222a4ddb079 2024-11-17T02:03:53,522 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-17T02:03:53,522 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-17T02:03:53,522 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-17T02:03:53,537 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-17T02:03:53,537 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-17T02:03:53,537 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-17T02:03:53,537 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-17T02:03:53,537 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-17T02:03:53,537 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-17T02:03:53,547 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-17T02:03:53,547 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-17T02:03:53,547 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-17T02:03:53,547 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@be1eab3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c6c0d7a0a7c0/172.17.0.2:0 2024-11-17T02:03:53,547 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3afe701c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c6c0d7a0a7c0/172.17.0.2:0 2024-11-17T02:03:53,547 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62e65a56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c6c0d7a0a7c0/172.17.0.2:0 2024-11-17T02:03:53,560 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;c6c0d7a0a7c0:45037 2024-11-17T02:03:53,563 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;c6c0d7a0a7c0:34033 2024-11-17T02:03:53,564 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c6c0d7a0a7c0:33127 2024-11-17T02:03:53,564 INFO [RS:1;c6c0d7a0a7c0:45037 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-17T02:03:53,564 INFO [RS:2;c6c0d7a0a7c0:34033 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-17T02:03:53,565 INFO [RS:0;c6c0d7a0a7c0:33127 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-17T02:03:53,565 INFO [RS:1;c6c0d7a0a7c0:45037 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-17T02:03:53,565 INFO [RS:2;c6c0d7a0a7c0:34033 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-17T02:03:53,565 INFO [RS:0;c6c0d7a0a7c0:33127 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-17T02:03:53,565 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-17T02:03:53,565 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-17T02:03:53,565 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-17T02:03:53,568 INFO [RS:1;c6c0d7a0a7c0:45037 {}] regionserver.HRegionServer(2659): reportForDuty to master=c6c0d7a0a7c0,32803,1731809031530 with port=45037, startcode=1731809032394 2024-11-17T02:03:53,568 INFO [RS:2;c6c0d7a0a7c0:34033 {}] regionserver.HRegionServer(2659): reportForDuty to master=c6c0d7a0a7c0,32803,1731809031530 with port=34033, startcode=1731809032455 2024-11-17T02:03:53,568 INFO [RS:0;c6c0d7a0a7c0:33127 {}] regionserver.HRegionServer(2659): reportForDuty to master=c6c0d7a0a7c0,32803,1731809031530 with port=33127, startcode=1731809032263 2024-11-17T02:03:53,579 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-17T02:03:53,579 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-17T02:03:53,579 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-17T02:03:53,591 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-17T02:03:53,602 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-17T02:03:53,610 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-17T02:03:53,618 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46655, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-17T02:03:53,618 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59959, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-17T02:03:53,618 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41215, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-17T02:03:53,618 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c6c0d7a0a7c0,32803,1731809031530 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-17T02:03:53,625 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32803 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:53,626 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c6c0d7a0a7c0:0, corePoolSize=5, maxPoolSize=5 2024-11-17T02:03:53,626 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c6c0d7a0a7c0:0, corePoolSize=5, maxPoolSize=5 2024-11-17T02:03:53,626 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c6c0d7a0a7c0:0, corePoolSize=5, maxPoolSize=5 2024-11-17T02:03:53,626 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c6c0d7a0a7c0:0, corePoolSize=5, maxPoolSize=5 2024-11-17T02:03:53,626 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c6c0d7a0a7c0:0, corePoolSize=10, maxPoolSize=10 2024-11-17T02:03:53,627 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T02:03:53,627 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c6c0d7a0a7c0:0, corePoolSize=2, maxPoolSize=2 2024-11-17T02:03:53,627 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T02:03:53,629 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731809063629 2024-11-17T02:03:53,630 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-17T02:03:53,631 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-17T02:03:53,632 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32803 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:53,633 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32803 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:53,635 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T02:03:53,636 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-17T02:03:53,636 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-17T02:03:53,636 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-17T02:03:53,636 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-17T02:03:53,637 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-17T02:03:53,637 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-17T02:03:53,641 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-17T02:03:53,642 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-17T02:03:53,643 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-17T02:03:53,643 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:53,643 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-17T02:03:53,645 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-17T02:03:53,645 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-17T02:03:53,647 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c6c0d7a0a7c0:0:becomeActiveMaster-HFileCleaner.large.0-1731809033647,5,FailOnTimeoutGroup] 2024-11-17T02:03:53,649 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c6c0d7a0a7c0:0:becomeActiveMaster-HFileCleaner.small.0-1731809033648,5,FailOnTimeoutGroup] 2024-11-17T02:03:53,649 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-17T02:03:53,649 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-17T02:03:53,651 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-17T02:03:53,651 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-17T02:03:53,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741831_1007 (size=1321) 2024-11-17T02:03:53,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741831_1007 (size=1321) 2024-11-17T02:03:53,673 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-17T02:03:53,673 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-17T02:03:53,673 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-17T02:03:53,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741831_1007 (size=1321) 2024-11-17T02:03:53,673 WARN [RS:1;c6c0d7a0a7c0:45037 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-17T02:03:53,673 WARN [RS:2;c6c0d7a0a7c0:34033 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-17T02:03:53,673 WARN [RS:0;c6c0d7a0a7c0:33127 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-17T02:03:53,675 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-17T02:03:53,676 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe 2024-11-17T02:03:53,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741832_1008 (size=32) 2024-11-17T02:03:53,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741832_1008 (size=32) 2024-11-17T02:03:53,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741832_1008 (size=32) 2024-11-17T02:03:53,691 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:53,694 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-17T02:03:53,697 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-17T02:03:53,697 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:53,698 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T02:03:53,698 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-17T02:03:53,701 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-17T02:03:53,701 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:53,702 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T02:03:53,702 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-17T02:03:53,705 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-17T02:03:53,705 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:53,706 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T02:03:53,707 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-17T02:03:53,710 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-17T02:03:53,710 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:53,711 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T02:03:53,711 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-17T02:03:53,713 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/hbase/meta/1588230740 2024-11-17T02:03:53,713 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/hbase/meta/1588230740 2024-11-17T02:03:53,716 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-17T02:03:53,717 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-17T02:03:53,718 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-17T02:03:53,720 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-17T02:03:53,724 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T02:03:53,726 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68571091, jitterRate=0.021788880228996277}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-17T02:03:53,730 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731809033691Initializing all the Stores at 1731809033693 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731809033693Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731809033694 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731809033694Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731809033694Cleaning up temporary data from old regions at 1731809033717 (+23 ms)Region opened successfully at 1731809033730 (+13 ms) 2024-11-17T02:03:53,730 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-17T02:03:53,730 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-17T02:03:53,731 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-17T02:03:53,731 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-17T02:03:53,731 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-17T02:03:53,733 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-17T02:03:53,733 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731809033730Disabling compacts and flushes for region at 1731809033730Disabling writes for close at 1731809033731 (+1 ms)Writing region close event to WAL at 1731809033732 (+1 ms)Closed at 1731809033733 (+1 ms) 2024-11-17T02:03:53,737 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T02:03:53,737 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-17T02:03:53,743 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-17T02:03:53,754 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T02:03:53,757 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-17T02:03:53,775 INFO [RS:0;c6c0d7a0a7c0:33127 {}] regionserver.HRegionServer(2659): reportForDuty to master=c6c0d7a0a7c0,32803,1731809031530 with port=33127, startcode=1731809032263 2024-11-17T02:03:53,775 INFO [RS:2;c6c0d7a0a7c0:34033 {}] regionserver.HRegionServer(2659): reportForDuty to master=c6c0d7a0a7c0,32803,1731809031530 with port=34033, startcode=1731809032455 2024-11-17T02:03:53,775 INFO [RS:1;c6c0d7a0a7c0:45037 {}] regionserver.HRegionServer(2659): reportForDuty to master=c6c0d7a0a7c0,32803,1731809031530 with port=45037, startcode=1731809032394 2024-11-17T02:03:53,776 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32803 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:53,779 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32803 {}] master.ServerManager(517): Registering regionserver=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:53,786 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32803 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:53,787 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32803 {}] master.ServerManager(517): Registering regionserver=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:53,787 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe 2024-11-17T02:03:53,787 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40353 2024-11-17T02:03:53,787 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-17T02:03:53,789 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32803 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c6c0d7a0a7c0,34033,1731809032455 2024-11-17T02:03:53,789 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe 2024-11-17T02:03:53,789 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32803 {}] master.ServerManager(517): Registering regionserver=c6c0d7a0a7c0,34033,1731809032455 2024-11-17T02:03:53,789 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40353 2024-11-17T02:03:53,789 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-17T02:03:53,792 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe 2024-11-17T02:03:53,792 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40353 2024-11-17T02:03:53,792 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-17T02:03:53,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-17T02:03:53,891 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] zookeeper.ZKUtil(111): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:53,893 WARN [RS:1;c6c0d7a0a7c0:45037 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-17T02:03:53,893 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] zookeeper.ZKUtil(111): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:53,893 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] zookeeper.ZKUtil(111): regionserver:34033-0x10147d245010003, quorum=127.0.0.1:53449, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c6c0d7a0a7c0,34033,1731809032455 2024-11-17T02:03:53,893 WARN [RS:0;c6c0d7a0a7c0:33127 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-17T02:03:53,893 INFO [RS:1;c6c0d7a0a7c0:45037 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T02:03:53,894 WARN [RS:2;c6c0d7a0a7c0:34033 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-17T02:03:53,894 INFO [RS:0;c6c0d7a0a7c0:33127 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T02:03:53,894 INFO [RS:2;c6c0d7a0a7c0:34033 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T02:03:53,894 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:53,895 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:53,895 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,34033,1731809032455 2024-11-17T02:03:53,897 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c6c0d7a0a7c0,45037,1731809032394] 2024-11-17T02:03:53,897 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c6c0d7a0a7c0,33127,1731809032263] 2024-11-17T02:03:53,898 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c6c0d7a0a7c0,34033,1731809032455] 2024-11-17T02:03:53,908 WARN [c6c0d7a0a7c0:32803 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-17T02:03:53,919 INFO [RS:0;c6c0d7a0a7c0:33127 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-17T02:03:53,919 INFO [RS:1;c6c0d7a0a7c0:45037 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-17T02:03:53,919 INFO [RS:2;c6c0d7a0a7c0:34033 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-17T02:03:53,935 INFO [RS:1;c6c0d7a0a7c0:45037 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-17T02:03:53,935 INFO [RS:2;c6c0d7a0a7c0:34033 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-17T02:03:53,935 INFO [RS:0;c6c0d7a0a7c0:33127 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-17T02:03:53,940 INFO [RS:1;c6c0d7a0a7c0:45037 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-17T02:03:53,940 INFO [RS:0;c6c0d7a0a7c0:33127 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-17T02:03:53,940 INFO [RS:2;c6c0d7a0a7c0:34033 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-17T02:03:53,940 INFO [RS:1;c6c0d7a0a7c0:45037 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T02:03:53,940 INFO [RS:2;c6c0d7a0a7c0:34033 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T02:03:53,940 INFO [RS:0;c6c0d7a0a7c0:33127 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T02:03:53,941 INFO [RS:0;c6c0d7a0a7c0:33127 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-17T02:03:53,941 INFO [RS:2;c6c0d7a0a7c0:34033 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-17T02:03:53,942 INFO [RS:1;c6c0d7a0a7c0:45037 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-17T02:03:53,947 INFO [RS:2;c6c0d7a0a7c0:34033 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-17T02:03:53,947 INFO [RS:1;c6c0d7a0a7c0:45037 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-17T02:03:53,947 INFO [RS:0;c6c0d7a0a7c0:33127 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-17T02:03:53,948 INFO [RS:0;c6c0d7a0a7c0:33127 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-17T02:03:53,948 INFO [RS:1;c6c0d7a0a7c0:45037 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-17T02:03:53,948 INFO [RS:2;c6c0d7a0a7c0:34033 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-17T02:03:53,949 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T02:03:53,949 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T02:03:53,949 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T02:03:53,949 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T02:03:53,949 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T02:03:53,949 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T02:03:53,949 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T02:03:53,949 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T02:03:53,949 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T02:03:53,949 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T02:03:53,949 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T02:03:53,949 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T02:03:53,949 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T02:03:53,949 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T02:03:53,949 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0, corePoolSize=2, maxPoolSize=2 2024-11-17T02:03:53,949 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0, corePoolSize=2, maxPoolSize=2 2024-11-17T02:03:53,950 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T02:03:53,950 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T02:03:53,950 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T02:03:53,950 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T02:03:53,950 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T02:03:53,950 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0, corePoolSize=2, maxPoolSize=2 2024-11-17T02:03:53,950 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T02:03:53,950 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T02:03:53,950 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T02:03:53,950 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T02:03:53,950 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T02:03:53,950 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T02:03:53,950 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T02:03:53,950 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T02:03:53,950 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T02:03:53,950 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T02:03:53,950 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c6c0d7a0a7c0:0, corePoolSize=3, maxPoolSize=3 2024-11-17T02:03:53,950 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c6c0d7a0a7c0:0, corePoolSize=3, maxPoolSize=3 2024-11-17T02:03:53,950 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T02:03:53,950 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0, corePoolSize=3, maxPoolSize=3 2024-11-17T02:03:53,950 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0, corePoolSize=3, maxPoolSize=3 2024-11-17T02:03:53,950 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T02:03:53,951 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T02:03:53,951 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T02:03:53,951 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c6c0d7a0a7c0:0, corePoolSize=3, maxPoolSize=3 2024-11-17T02:03:53,951 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0, corePoolSize=3, maxPoolSize=3 2024-11-17T02:03:53,955 INFO [RS:0;c6c0d7a0a7c0:33127 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T02:03:53,955 INFO [RS:2;c6c0d7a0a7c0:34033 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T02:03:53,955 INFO [RS:0;c6c0d7a0a7c0:33127 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T02:03:53,955 INFO [RS:1;c6c0d7a0a7c0:45037 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T02:03:53,955 INFO [RS:2;c6c0d7a0a7c0:34033 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T02:03:53,955 INFO [RS:0;c6c0d7a0a7c0:33127 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T02:03:53,955 INFO [RS:1;c6c0d7a0a7c0:45037 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T02:03:53,955 INFO [RS:2;c6c0d7a0a7c0:34033 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T02:03:53,955 INFO [RS:0;c6c0d7a0a7c0:33127 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-17T02:03:53,955 INFO [RS:1;c6c0d7a0a7c0:45037 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T02:03:53,955 INFO [RS:2;c6c0d7a0a7c0:34033 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-17T02:03:53,955 INFO [RS:0;c6c0d7a0a7c0:33127 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-17T02:03:53,955 INFO [RS:2;c6c0d7a0a7c0:34033 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-17T02:03:53,955 INFO [RS:1;c6c0d7a0a7c0:45037 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-17T02:03:53,955 INFO [RS:0;c6c0d7a0a7c0:33127 {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,33127,1731809032263-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-17T02:03:53,955 INFO [RS:2;c6c0d7a0a7c0:34033 {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,34033,1731809032455-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-17T02:03:53,955 INFO [RS:1;c6c0d7a0a7c0:45037 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-17T02:03:53,956 INFO [RS:1;c6c0d7a0a7c0:45037 {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,45037,1731809032394-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-17T02:03:53,974 INFO [RS:0;c6c0d7a0a7c0:33127 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-17T02:03:53,974 INFO [RS:2;c6c0d7a0a7c0:34033 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-17T02:03:53,974 INFO [RS:1;c6c0d7a0a7c0:45037 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-17T02:03:53,976 INFO [RS:0;c6c0d7a0a7c0:33127 {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,33127,1731809032263-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T02:03:53,976 INFO [RS:1;c6c0d7a0a7c0:45037 {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,45037,1731809032394-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T02:03:53,976 INFO [RS:2;c6c0d7a0a7c0:34033 {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,34033,1731809032455-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T02:03:53,976 INFO [RS:0;c6c0d7a0a7c0:33127 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T02:03:53,976 INFO [RS:1;c6c0d7a0a7c0:45037 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T02:03:53,976 INFO [RS:2;c6c0d7a0a7c0:34033 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T02:03:53,976 INFO [RS:1;c6c0d7a0a7c0:45037 {}] regionserver.Replication(171): c6c0d7a0a7c0,45037,1731809032394 started 2024-11-17T02:03:53,976 INFO [RS:2;c6c0d7a0a7c0:34033 {}] regionserver.Replication(171): c6c0d7a0a7c0,34033,1731809032455 started 2024-11-17T02:03:53,976 INFO [RS:0;c6c0d7a0a7c0:33127 {}] regionserver.Replication(171): c6c0d7a0a7c0,33127,1731809032263 started 2024-11-17T02:03:53,993 INFO [RS:0;c6c0d7a0a7c0:33127 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T02:03:53,993 INFO [RS:1;c6c0d7a0a7c0:45037 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T02:03:53,994 INFO [RS:0;c6c0d7a0a7c0:33127 {}] regionserver.HRegionServer(1482): Serving as c6c0d7a0a7c0,33127,1731809032263, RpcServer on c6c0d7a0a7c0/172.17.0.2:33127, sessionid=0x10147d245010001 2024-11-17T02:03:53,994 INFO [RS:1;c6c0d7a0a7c0:45037 {}] regionserver.HRegionServer(1482): Serving as c6c0d7a0a7c0,45037,1731809032394, RpcServer on c6c0d7a0a7c0/172.17.0.2:45037, sessionid=0x10147d245010002 2024-11-17T02:03:53,994 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-17T02:03:53,994 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-17T02:03:53,994 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:53,994 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:53,995 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c6c0d7a0a7c0,33127,1731809032263' 2024-11-17T02:03:53,995 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c6c0d7a0a7c0,45037,1731809032394' 2024-11-17T02:03:53,995 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-17T02:03:53,995 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-17T02:03:53,996 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-17T02:03:53,996 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-17T02:03:53,996 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-17T02:03:53,996 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-17T02:03:53,996 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-17T02:03:53,996 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-17T02:03:53,997 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:53,997 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:53,997 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c6c0d7a0a7c0,33127,1731809032263' 2024-11-17T02:03:53,997 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c6c0d7a0a7c0,45037,1731809032394' 2024-11-17T02:03:53,997 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-17T02:03:53,997 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-17T02:03:53,997 INFO [RS:2;c6c0d7a0a7c0:34033 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T02:03:53,997 INFO [RS:2;c6c0d7a0a7c0:34033 {}] regionserver.HRegionServer(1482): Serving as c6c0d7a0a7c0,34033,1731809032455, RpcServer on c6c0d7a0a7c0/172.17.0.2:34033, sessionid=0x10147d245010003 2024-11-17T02:03:53,997 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-17T02:03:53,997 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c6c0d7a0a7c0,34033,1731809032455 2024-11-17T02:03:53,997 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-17T02:03:53,997 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c6c0d7a0a7c0,34033,1731809032455' 2024-11-17T02:03:53,997 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-17T02:03:53,997 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-17T02:03:53,998 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-17T02:03:53,998 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-17T02:03:53,998 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-17T02:03:53,998 INFO [RS:1;c6c0d7a0a7c0:45037 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-17T02:03:53,998 INFO [RS:0;c6c0d7a0a7c0:33127 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-17T02:03:53,998 INFO [RS:1;c6c0d7a0a7c0:45037 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-17T02:03:53,998 INFO [RS:0;c6c0d7a0a7c0:33127 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-17T02:03:53,998 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-17T02:03:53,998 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-17T02:03:53,999 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c6c0d7a0a7c0,34033,1731809032455 2024-11-17T02:03:53,999 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c6c0d7a0a7c0,34033,1731809032455' 2024-11-17T02:03:53,999 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-17T02:03:53,999 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-17T02:03:54,000 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-17T02:03:54,000 INFO [RS:2;c6c0d7a0a7c0:34033 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-17T02:03:54,000 INFO [RS:2;c6c0d7a0a7c0:34033 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-17T02:03:54,109 INFO [RS:2;c6c0d7a0a7c0:34033 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T02:03:54,109 INFO [RS:1;c6c0d7a0a7c0:45037 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T02:03:54,109 INFO [RS:0;c6c0d7a0a7c0:33127 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T02:03:54,112 INFO [RS:1;c6c0d7a0a7c0:45037 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c6c0d7a0a7c0%2C45037%2C1731809032394, suffix=, logDir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,45037,1731809032394, archiveDir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/oldWALs, maxLogs=32 2024-11-17T02:03:54,112 INFO [RS:2;c6c0d7a0a7c0:34033 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c6c0d7a0a7c0%2C34033%2C1731809032455, suffix=, logDir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,34033,1731809032455, archiveDir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/oldWALs, maxLogs=32 2024-11-17T02:03:54,112 INFO [RS:0;c6c0d7a0a7c0:33127 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c6c0d7a0a7c0%2C33127%2C1731809032263, suffix=, logDir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,33127,1731809032263, archiveDir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/oldWALs, maxLogs=32 2024-11-17T02:03:54,128 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,45037,1731809032394/c6c0d7a0a7c0%2C45037%2C1731809032394.1731809034116, exclude list is [], retry=0 2024-11-17T02:03:54,128 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,33127,1731809032263/c6c0d7a0a7c0%2C33127%2C1731809032263.1731809034116, exclude list is [], retry=0 2024-11-17T02:03:54,128 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,34033,1731809032455/c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, exclude list is [], retry=0 2024-11-17T02:03:54,133 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36067,DS-11c5e779-9b67-42ae-934a-e223a96170e4,DISK] 2024-11-17T02:03:54,133 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33655,DS-19b795a5-ad52-4253-ba85-390185288d22,DISK] 2024-11-17T02:03:54,133 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36067,DS-11c5e779-9b67-42ae-934a-e223a96170e4,DISK] 2024-11-17T02:03:54,134 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36455,DS-da656d4a-0400-49ad-97b4-848c04dabe50,DISK] 2024-11-17T02:03:54,134 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33655,DS-19b795a5-ad52-4253-ba85-390185288d22,DISK] 2024-11-17T02:03:54,134 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33655,DS-19b795a5-ad52-4253-ba85-390185288d22,DISK] 2024-11-17T02:03:54,134 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36455,DS-da656d4a-0400-49ad-97b4-848c04dabe50,DISK] 2024-11-17T02:03:54,135 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36455,DS-da656d4a-0400-49ad-97b4-848c04dabe50,DISK] 2024-11-17T02:03:54,135 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36067,DS-11c5e779-9b67-42ae-934a-e223a96170e4,DISK] 2024-11-17T02:03:54,140 INFO [RS:0;c6c0d7a0a7c0:33127 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,33127,1731809032263/c6c0d7a0a7c0%2C33127%2C1731809032263.1731809034116 2024-11-17T02:03:54,140 INFO [RS:1;c6c0d7a0a7c0:45037 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,45037,1731809032394/c6c0d7a0a7c0%2C45037%2C1731809032394.1731809034116 2024-11-17T02:03:54,141 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40831:40831),(127.0.0.1/127.0.0.1:40369:40369),(127.0.0.1/127.0.0.1:37075:37075)] 2024-11-17T02:03:54,142 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40831:40831),(127.0.0.1/127.0.0.1:40369:40369),(127.0.0.1/127.0.0.1:37075:37075)] 2024-11-17T02:03:54,142 INFO [RS:2;c6c0d7a0a7c0:34033 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,34033,1731809032455/c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116 2024-11-17T02:03:54,142 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40369:40369),(127.0.0.1/127.0.0.1:37075:37075),(127.0.0.1/127.0.0.1:40831:40831)] 2024-11-17T02:03:54,160 DEBUG [c6c0d7a0a7c0:32803 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-17T02:03:54,167 DEBUG [c6c0d7a0a7c0:32803 {}] balancer.BalancerClusterState(204): Hosts are {c6c0d7a0a7c0=0} racks are {/default-rack=0} 2024-11-17T02:03:54,173 DEBUG [c6c0d7a0a7c0:32803 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-17T02:03:54,173 DEBUG [c6c0d7a0a7c0:32803 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-17T02:03:54,173 DEBUG [c6c0d7a0a7c0:32803 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-17T02:03:54,173 DEBUG [c6c0d7a0a7c0:32803 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-17T02:03:54,173 DEBUG [c6c0d7a0a7c0:32803 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-17T02:03:54,173 DEBUG [c6c0d7a0a7c0:32803 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-17T02:03:54,173 INFO [c6c0d7a0a7c0:32803 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-17T02:03:54,173 INFO [c6c0d7a0a7c0:32803 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-17T02:03:54,173 INFO [c6c0d7a0a7c0:32803 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-17T02:03:54,173 DEBUG [c6c0d7a0a7c0:32803 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-17T02:03:54,180 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c6c0d7a0a7c0,34033,1731809032455 2024-11-17T02:03:54,186 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c6c0d7a0a7c0,34033,1731809032455, state=OPENING 2024-11-17T02:03:54,220 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-17T02:03:54,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T02:03:54,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T02:03:54,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34033-0x10147d245010003, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T02:03:54,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T02:03:54,229 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:54,229 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:54,229 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:54,230 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:54,231 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T02:03:54,234 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,34033,1731809032455}] 2024-11-17T02:03:54,412 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-17T02:03:54,414 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41789, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-17T02:03:54,426 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-17T02:03:54,426 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T02:03:54,427 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-17T02:03:54,430 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c6c0d7a0a7c0%2C34033%2C1731809032455.meta, suffix=.meta, logDir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,34033,1731809032455, archiveDir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/oldWALs, maxLogs=32 2024-11-17T02:03:54,445 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,34033,1731809032455/c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, exclude list is [], retry=0 2024-11-17T02:03:54,449 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33655,DS-19b795a5-ad52-4253-ba85-390185288d22,DISK] 2024-11-17T02:03:54,449 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36455,DS-da656d4a-0400-49ad-97b4-848c04dabe50,DISK] 2024-11-17T02:03:54,449 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36067,DS-11c5e779-9b67-42ae-934a-e223a96170e4,DISK] 2024-11-17T02:03:54,452 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,34033,1731809032455/c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta 2024-11-17T02:03:54,452 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40369:40369),(127.0.0.1/127.0.0.1:37075:37075),(127.0.0.1/127.0.0.1:40831:40831)] 2024-11-17T02:03:54,452 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-17T02:03:54,454 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-17T02:03:54,456 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-17T02:03:54,461 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-17T02:03:54,464 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-17T02:03:54,465 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:54,465 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-17T02:03:54,465 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-17T02:03:54,469 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-17T02:03:54,470 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-17T02:03:54,471 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:54,472 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T02:03:54,472 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-17T02:03:54,473 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-17T02:03:54,473 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:54,475 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T02:03:54,475 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-17T02:03:54,476 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-17T02:03:54,476 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:54,477 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T02:03:54,478 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-17T02:03:54,479 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-17T02:03:54,479 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:54,480 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T02:03:54,480 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-17T02:03:54,481 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/hbase/meta/1588230740 2024-11-17T02:03:54,484 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/hbase/meta/1588230740 2024-11-17T02:03:54,486 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-17T02:03:54,486 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-17T02:03:54,487 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-17T02:03:54,490 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-17T02:03:54,491 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74954318, jitterRate=0.11690637469291687}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-17T02:03:54,491 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-17T02:03:54,493 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731809034466Writing region info on filesystem at 1731809034466Initializing all the Stores at 1731809034468 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731809034468Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731809034469 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731809034469Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731809034469Cleaning up temporary data from old regions at 1731809034486 (+17 ms)Running coprocessor post-open hooks at 1731809034492 (+6 ms)Region opened successfully at 1731809034492 2024-11-17T02:03:54,499 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731809034403 2024-11-17T02:03:54,509 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-17T02:03:54,510 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-17T02:03:54,512 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,34033,1731809032455 2024-11-17T02:03:54,514 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c6c0d7a0a7c0,34033,1731809032455, state=OPEN 2024-11-17T02:03:54,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:54,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:54,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34033-0x10147d245010003, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:54,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:54,537 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:54,537 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:54,537 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:54,537 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:54,538 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,34033,1731809032455 2024-11-17T02:03:54,543 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-17T02:03:54,543 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,34033,1731809032455 in 304 msec 2024-11-17T02:03:54,550 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-17T02:03:54,551 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 802 msec 2024-11-17T02:03:54,552 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T02:03:54,552 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-17T02:03:54,571 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T02:03:54,572 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,34033,1731809032455, seqNum=-1] 2024-11-17T02:03:54,593 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T02:03:54,595 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59773, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T02:03:54,614 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0670 sec 2024-11-17T02:03:54,614 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731809034614, completionTime=-1 2024-11-17T02:03:54,617 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-17T02:03:54,617 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-17T02:03:54,677 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-17T02:03:54,678 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731809094678 2024-11-17T02:03:54,678 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731809154678 2024-11-17T02:03:54,678 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 60 msec 2024-11-17T02:03:54,680 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-17T02:03:54,691 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,32803,1731809031530-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T02:03:54,691 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,32803,1731809031530-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T02:03:54,691 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,32803,1731809031530-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T02:03:54,693 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c6c0d7a0a7c0:32803, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T02:03:54,694 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-17T02:03:54,694 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-17T02:03:54,701 DEBUG [master/c6c0d7a0a7c0:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-17T02:03:54,725 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.121sec 2024-11-17T02:03:54,727 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-17T02:03:54,729 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-17T02:03:54,729 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-17T02:03:54,730 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-17T02:03:54,730 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-17T02:03:54,731 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,32803,1731809031530-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-17T02:03:54,731 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,32803,1731809031530-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-17T02:03:54,735 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-17T02:03:54,736 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-17T02:03:54,737 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,32803,1731809031530-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T02:03:54,828 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9330f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T02:03:54,832 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-17T02:03:54,832 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-17T02:03:54,835 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,32803,-1 for getting cluster id 2024-11-17T02:03:54,837 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T02:03:54,844 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4e2710dd-e280-4e22-bcbe-5222a4ddb079' 2024-11-17T02:03:54,847 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T02:03:54,847 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4e2710dd-e280-4e22-bcbe-5222a4ddb079" 2024-11-17T02:03:54,848 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@131ce487, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T02:03:54,848 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,32803,-1] 2024-11-17T02:03:54,850 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T02:03:54,852 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T02:03:54,854 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51558, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T02:03:54,856 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6982d266, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T02:03:54,856 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T02:03:54,863 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,34033,1731809032455, seqNum=-1] 2024-11-17T02:03:54,864 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T02:03:54,867 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57754, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T02:03:54,885 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c6c0d7a0a7c0,32803,1731809031530 2024-11-17T02:03:54,888 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T02:03:54,893 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is c6c0d7a0a7c0,32803,1731809031530 2024-11-17T02:03:54,895 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3ae6b2d 2024-11-17T02:03:54,896 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T02:03:54,898 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51562, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T02:03:54,905 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T02:03:54,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-11-17T02:03:54,917 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T02:03:54,919 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-11-17T02:03:54,919 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:54,922 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T02:03:54,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T02:03:54,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741837_1013 (size=358) 2024-11-17T02:03:54,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741837_1013 (size=358) 2024-11-17T02:03:54,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741837_1013 (size=358) 2024-11-17T02:03:54,941 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => a17ac9e270344fa2752f51f8196693b2, NAME => 'test,bbb,1731809034900.a17ac9e270344fa2752f51f8196693b2.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe 2024-11-17T02:03:54,941 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => de9924be32211cddceef43e5b4866dc8, NAME => 'test,,1731809034900.de9924be32211cddceef43e5b4866dc8.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe 2024-11-17T02:03:54,942 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 81f322eb57dcb42c3b4269a738bafd6d, NAME => 'test,ccc,1731809034900.81f322eb57dcb42c3b4269a738bafd6d.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe 2024-11-17T02:03:54,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741838_1014 (size=45) 2024-11-17T02:03:54,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741838_1014 (size=45) 2024-11-17T02:03:54,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741838_1014 (size=45) 2024-11-17T02:03:54,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741839_1015 (size=45) 2024-11-17T02:03:54,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741839_1015 (size=45) 2024-11-17T02:03:54,966 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1731809034900.a17ac9e270344fa2752f51f8196693b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:54,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741839_1015 (size=45) 2024-11-17T02:03:54,966 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing a17ac9e270344fa2752f51f8196693b2, disabling compactions & flushes 2024-11-17T02:03:54,966 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1731809034900.a17ac9e270344fa2752f51f8196693b2. 2024-11-17T02:03:54,966 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1731809034900.a17ac9e270344fa2752f51f8196693b2. 2024-11-17T02:03:54,966 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1731809034900.a17ac9e270344fa2752f51f8196693b2. after waiting 0 ms 2024-11-17T02:03:54,966 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1731809034900.a17ac9e270344fa2752f51f8196693b2. 2024-11-17T02:03:54,966 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1731809034900.a17ac9e270344fa2752f51f8196693b2. 2024-11-17T02:03:54,966 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for a17ac9e270344fa2752f51f8196693b2: Waiting for close lock at 1731809034966Disabling compacts and flushes for region at 1731809034966Disabling writes for close at 1731809034966Writing region close event to WAL at 1731809034966Closed at 1731809034966 2024-11-17T02:03:54,967 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 15c64fab21a3ca03ec0a7afabc19cdc9, NAME => 'test,ddd,1731809034900.15c64fab21a3ca03ec0a7afabc19cdc9.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe 2024-11-17T02:03:54,967 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1731809034900.81f322eb57dcb42c3b4269a738bafd6d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:54,967 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 81f322eb57dcb42c3b4269a738bafd6d, disabling compactions & flushes 2024-11-17T02:03:54,967 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1731809034900.81f322eb57dcb42c3b4269a738bafd6d. 2024-11-17T02:03:54,967 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1731809034900.81f322eb57dcb42c3b4269a738bafd6d. 2024-11-17T02:03:54,967 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1731809034900.81f322eb57dcb42c3b4269a738bafd6d. after waiting 0 ms 2024-11-17T02:03:54,967 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1731809034900.81f322eb57dcb42c3b4269a738bafd6d. 2024-11-17T02:03:54,967 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1731809034900.81f322eb57dcb42c3b4269a738bafd6d. 2024-11-17T02:03:54,968 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 81f322eb57dcb42c3b4269a738bafd6d: Waiting for close lock at 1731809034967Disabling compacts and flushes for region at 1731809034967Disabling writes for close at 1731809034967Writing region close event to WAL at 1731809034967Closed at 1731809034967 2024-11-17T02:03:54,968 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 8356c0fdb856c376b8678f2c2597565a, NAME => 'test,eee,1731809034900.8356c0fdb856c376b8678f2c2597565a.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe 2024-11-17T02:03:54,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741840_1016 (size=42) 2024-11-17T02:03:54,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741840_1016 (size=42) 2024-11-17T02:03:54,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741840_1016 (size=42) 2024-11-17T02:03:54,979 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1731809034900.de9924be32211cddceef43e5b4866dc8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:54,980 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing de9924be32211cddceef43e5b4866dc8, disabling compactions & flushes 2024-11-17T02:03:54,980 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1731809034900.de9924be32211cddceef43e5b4866dc8. 2024-11-17T02:03:54,980 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1731809034900.de9924be32211cddceef43e5b4866dc8. 2024-11-17T02:03:54,980 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1731809034900.de9924be32211cddceef43e5b4866dc8. after waiting 0 ms 2024-11-17T02:03:54,980 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1731809034900.de9924be32211cddceef43e5b4866dc8. 2024-11-17T02:03:54,980 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1731809034900.de9924be32211cddceef43e5b4866dc8. 2024-11-17T02:03:54,980 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for de9924be32211cddceef43e5b4866dc8: Waiting for close lock at 1731809034980Disabling compacts and flushes for region at 1731809034980Disabling writes for close at 1731809034980Writing region close event to WAL at 1731809034980Closed at 1731809034980 2024-11-17T02:03:54,981 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 112f5bbe15f72793585db9ba98950976, NAME => 'test,fff,1731809034900.112f5bbe15f72793585db9ba98950976.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe 2024-11-17T02:03:54,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741841_1017 (size=45) 2024-11-17T02:03:54,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741841_1017 (size=45) 2024-11-17T02:03:54,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741841_1017 (size=45) 2024-11-17T02:03:54,995 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ddd,1731809034900.15c64fab21a3ca03ec0a7afabc19cdc9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:54,995 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 15c64fab21a3ca03ec0a7afabc19cdc9, disabling compactions & flushes 2024-11-17T02:03:54,995 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ddd,1731809034900.15c64fab21a3ca03ec0a7afabc19cdc9. 2024-11-17T02:03:54,996 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1731809034900.15c64fab21a3ca03ec0a7afabc19cdc9. 2024-11-17T02:03:54,996 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1731809034900.15c64fab21a3ca03ec0a7afabc19cdc9. after waiting 0 ms 2024-11-17T02:03:54,996 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1731809034900.15c64fab21a3ca03ec0a7afabc19cdc9. 2024-11-17T02:03:54,996 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ddd,1731809034900.15c64fab21a3ca03ec0a7afabc19cdc9. 2024-11-17T02:03:54,996 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 15c64fab21a3ca03ec0a7afabc19cdc9: Waiting for close lock at 1731809034995Disabling compacts and flushes for region at 1731809034995Disabling writes for close at 1731809034996 (+1 ms)Writing region close event to WAL at 1731809034996Closed at 1731809034996 2024-11-17T02:03:54,997 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 7517bce9e04879764302cccd45af20fb, NAME => 'test,ggg,1731809034900.7517bce9e04879764302cccd45af20fb.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe 2024-11-17T02:03:54,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741842_1018 (size=45) 2024-11-17T02:03:54,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741842_1018 (size=45) 2024-11-17T02:03:55,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741842_1018 (size=45) 2024-11-17T02:03:55,001 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,eee,1731809034900.8356c0fdb856c376b8678f2c2597565a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:55,001 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 8356c0fdb856c376b8678f2c2597565a, disabling compactions & flushes 2024-11-17T02:03:55,001 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,eee,1731809034900.8356c0fdb856c376b8678f2c2597565a. 2024-11-17T02:03:55,001 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1731809034900.8356c0fdb856c376b8678f2c2597565a. 2024-11-17T02:03:55,001 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1731809034900.8356c0fdb856c376b8678f2c2597565a. after waiting 0 ms 2024-11-17T02:03:55,001 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1731809034900.8356c0fdb856c376b8678f2c2597565a. 2024-11-17T02:03:55,001 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,eee,1731809034900.8356c0fdb856c376b8678f2c2597565a. 2024-11-17T02:03:55,001 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 8356c0fdb856c376b8678f2c2597565a: Waiting for close lock at 1731809035001Disabling compacts and flushes for region at 1731809035001Disabling writes for close at 1731809035001Writing region close event to WAL at 1731809035001Closed at 1731809035001 2024-11-17T02:03:55,002 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 6fd33da09f9d938215ac0e7127ad70d5, NAME => 'test,hhh,1731809034900.6fd33da09f9d938215ac0e7127ad70d5.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe 2024-11-17T02:03:55,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741843_1019 (size=45) 2024-11-17T02:03:55,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741844_1020 (size=45) 2024-11-17T02:03:55,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741843_1019 (size=45) 2024-11-17T02:03:55,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741844_1020 (size=45) 2024-11-17T02:03:55,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741845_1021 (size=45) 2024-11-17T02:03:55,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741844_1020 (size=45) 2024-11-17T02:03:55,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741843_1019 (size=45) 2024-11-17T02:03:55,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741845_1021 (size=45) 2024-11-17T02:03:55,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741845_1021 (size=45) 2024-11-17T02:03:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T02:03:55,031 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,hhh,1731809034900.6fd33da09f9d938215ac0e7127ad70d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:55,031 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,fff,1731809034900.112f5bbe15f72793585db9ba98950976.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:55,031 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 112f5bbe15f72793585db9ba98950976, disabling compactions & flushes 2024-11-17T02:03:55,031 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 6fd33da09f9d938215ac0e7127ad70d5, disabling compactions & flushes 2024-11-17T02:03:55,031 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,fff,1731809034900.112f5bbe15f72793585db9ba98950976. 2024-11-17T02:03:55,031 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1731809034900.112f5bbe15f72793585db9ba98950976. 2024-11-17T02:03:55,031 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,hhh,1731809034900.6fd33da09f9d938215ac0e7127ad70d5. 2024-11-17T02:03:55,031 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1731809034900.6fd33da09f9d938215ac0e7127ad70d5. 2024-11-17T02:03:55,031 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1731809034900.112f5bbe15f72793585db9ba98950976. after waiting 0 ms 2024-11-17T02:03:55,031 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1731809034900.6fd33da09f9d938215ac0e7127ad70d5. after waiting 0 ms 2024-11-17T02:03:55,031 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1731809034900.112f5bbe15f72793585db9ba98950976. 2024-11-17T02:03:55,031 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1731809034900.6fd33da09f9d938215ac0e7127ad70d5. 2024-11-17T02:03:55,031 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,fff,1731809034900.112f5bbe15f72793585db9ba98950976. 2024-11-17T02:03:55,031 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,hhh,1731809034900.6fd33da09f9d938215ac0e7127ad70d5. 2024-11-17T02:03:55,031 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 112f5bbe15f72793585db9ba98950976: Waiting for close lock at 1731809035031Disabling compacts and flushes for region at 1731809035031Disabling writes for close at 1731809035031Writing region close event to WAL at 1731809035031Closed at 1731809035031 2024-11-17T02:03:55,031 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 6fd33da09f9d938215ac0e7127ad70d5: Waiting for close lock at 1731809035031Disabling compacts and flushes for region at 1731809035031Disabling writes for close at 1731809035031Writing region close event to WAL at 1731809035031Closed at 1731809035031 2024-11-17T02:03:55,032 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 03b7adb3afa9bebd86dc1da38125b982, NAME => 'test,iii,1731809034900.03b7adb3afa9bebd86dc1da38125b982.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe 2024-11-17T02:03:55,032 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 05534a4a027a1fdc90227b7a2e681933, NAME => 'test,jjj,1731809034900.05534a4a027a1fdc90227b7a2e681933.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe 2024-11-17T02:03:55,033 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ggg,1731809034900.7517bce9e04879764302cccd45af20fb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:55,033 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 7517bce9e04879764302cccd45af20fb, disabling compactions & flushes 2024-11-17T02:03:55,033 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ggg,1731809034900.7517bce9e04879764302cccd45af20fb. 2024-11-17T02:03:55,033 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1731809034900.7517bce9e04879764302cccd45af20fb. 2024-11-17T02:03:55,033 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1731809034900.7517bce9e04879764302cccd45af20fb. after waiting 0 ms 2024-11-17T02:03:55,033 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1731809034900.7517bce9e04879764302cccd45af20fb. 2024-11-17T02:03:55,033 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ggg,1731809034900.7517bce9e04879764302cccd45af20fb. 2024-11-17T02:03:55,033 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 7517bce9e04879764302cccd45af20fb: Waiting for close lock at 1731809035033Disabling compacts and flushes for region at 1731809035033Disabling writes for close at 1731809035033Writing region close event to WAL at 1731809035033Closed at 1731809035033 2024-11-17T02:03:55,034 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => ac10d67ff6c3f527cc0861e738032bdb, NAME => 'test,kkk,1731809034900.ac10d67ff6c3f527cc0861e738032bdb.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe 2024-11-17T02:03:55,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741847_1023 (size=45) 2024-11-17T02:03:55,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741847_1023 (size=45) 2024-11-17T02:03:55,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741847_1023 (size=45) 2024-11-17T02:03:55,047 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,iii,1731809034900.03b7adb3afa9bebd86dc1da38125b982.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:55,047 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 03b7adb3afa9bebd86dc1da38125b982, disabling compactions & flushes 2024-11-17T02:03:55,047 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,iii,1731809034900.03b7adb3afa9bebd86dc1da38125b982. 2024-11-17T02:03:55,047 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1731809034900.03b7adb3afa9bebd86dc1da38125b982. 2024-11-17T02:03:55,047 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1731809034900.03b7adb3afa9bebd86dc1da38125b982. after waiting 0 ms 2024-11-17T02:03:55,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741848_1024 (size=45) 2024-11-17T02:03:55,048 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1731809034900.03b7adb3afa9bebd86dc1da38125b982. 2024-11-17T02:03:55,048 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,iii,1731809034900.03b7adb3afa9bebd86dc1da38125b982. 2024-11-17T02:03:55,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741846_1022 (size=45) 2024-11-17T02:03:55,048 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 03b7adb3afa9bebd86dc1da38125b982: Waiting for close lock at 1731809035047Disabling compacts and flushes for region at 1731809035047Disabling writes for close at 1731809035048 (+1 ms)Writing region close event to WAL at 1731809035048Closed at 1731809035048 2024-11-17T02:03:55,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741848_1024 (size=45) 2024-11-17T02:03:55,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741848_1024 (size=45) 2024-11-17T02:03:55,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741846_1022 (size=45) 2024-11-17T02:03:55,049 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1056118c93716b24d18f749826cf316a, NAME => 'test,lll,1731809034900.1056118c93716b24d18f749826cf316a.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe 2024-11-17T02:03:55,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741846_1022 (size=45) 2024-11-17T02:03:55,050 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,kkk,1731809034900.ac10d67ff6c3f527cc0861e738032bdb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:55,050 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,jjj,1731809034900.05534a4a027a1fdc90227b7a2e681933.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:55,051 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 05534a4a027a1fdc90227b7a2e681933, disabling compactions & flushes 2024-11-17T02:03:55,051 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing ac10d67ff6c3f527cc0861e738032bdb, disabling compactions & flushes 2024-11-17T02:03:55,051 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,kkk,1731809034900.ac10d67ff6c3f527cc0861e738032bdb. 2024-11-17T02:03:55,051 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,jjj,1731809034900.05534a4a027a1fdc90227b7a2e681933. 2024-11-17T02:03:55,051 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1731809034900.ac10d67ff6c3f527cc0861e738032bdb. 2024-11-17T02:03:55,051 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1731809034900.05534a4a027a1fdc90227b7a2e681933. 2024-11-17T02:03:55,051 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1731809034900.ac10d67ff6c3f527cc0861e738032bdb. after waiting 0 ms 2024-11-17T02:03:55,051 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1731809034900.05534a4a027a1fdc90227b7a2e681933. after waiting 0 ms 2024-11-17T02:03:55,051 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1731809034900.ac10d67ff6c3f527cc0861e738032bdb. 2024-11-17T02:03:55,051 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1731809034900.05534a4a027a1fdc90227b7a2e681933. 2024-11-17T02:03:55,051 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,kkk,1731809034900.ac10d67ff6c3f527cc0861e738032bdb. 2024-11-17T02:03:55,051 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,jjj,1731809034900.05534a4a027a1fdc90227b7a2e681933. 2024-11-17T02:03:55,051 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for ac10d67ff6c3f527cc0861e738032bdb: Waiting for close lock at 1731809035051Disabling compacts and flushes for region at 1731809035051Disabling writes for close at 1731809035051Writing region close event to WAL at 1731809035051Closed at 1731809035051 2024-11-17T02:03:55,051 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 05534a4a027a1fdc90227b7a2e681933: Waiting for close lock at 1731809035051Disabling compacts and flushes for region at 1731809035051Disabling writes for close at 1731809035051Writing region close event to WAL at 1731809035051Closed at 1731809035051 2024-11-17T02:03:55,052 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 663bac36df6c9873ad776bb3ad42409f, NAME => 'test,mmm,1731809034900.663bac36df6c9873ad776bb3ad42409f.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe 2024-11-17T02:03:55,052 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 74108f0b014075d4993f0eee396e39e5, NAME => 'test,nnn,1731809034900.74108f0b014075d4993f0eee396e39e5.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe 2024-11-17T02:03:55,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741849_1025 (size=45) 2024-11-17T02:03:55,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741849_1025 (size=45) 2024-11-17T02:03:55,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741849_1025 (size=45) 2024-11-17T02:03:55,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741851_1027 (size=45) 2024-11-17T02:03:55,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741851_1027 (size=45) 2024-11-17T02:03:55,065 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,lll,1731809034900.1056118c93716b24d18f749826cf316a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:55,066 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 1056118c93716b24d18f749826cf316a, disabling compactions & flushes 2024-11-17T02:03:55,066 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,lll,1731809034900.1056118c93716b24d18f749826cf316a. 2024-11-17T02:03:55,066 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1731809034900.1056118c93716b24d18f749826cf316a. 2024-11-17T02:03:55,066 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1731809034900.1056118c93716b24d18f749826cf316a. after waiting 0 ms 2024-11-17T02:03:55,066 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1731809034900.1056118c93716b24d18f749826cf316a. 2024-11-17T02:03:55,066 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,lll,1731809034900.1056118c93716b24d18f749826cf316a. 2024-11-17T02:03:55,066 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1056118c93716b24d18f749826cf316a: Waiting for close lock at 1731809035066Disabling compacts and flushes for region at 1731809035066Disabling writes for close at 1731809035066Writing region close event to WAL at 1731809035066Closed at 1731809035066 2024-11-17T02:03:55,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741851_1027 (size=45) 2024-11-17T02:03:55,067 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 25a67f85380b4a3cd6890ab81d0b3a27, NAME => 'test,ooo,1731809034900.25a67f85380b4a3cd6890ab81d0b3a27.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe 2024-11-17T02:03:55,067 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,nnn,1731809034900.74108f0b014075d4993f0eee396e39e5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:55,067 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 74108f0b014075d4993f0eee396e39e5, disabling compactions & flushes 2024-11-17T02:03:55,068 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,nnn,1731809034900.74108f0b014075d4993f0eee396e39e5. 2024-11-17T02:03:55,068 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1731809034900.74108f0b014075d4993f0eee396e39e5. 2024-11-17T02:03:55,068 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1731809034900.74108f0b014075d4993f0eee396e39e5. after waiting 0 ms 2024-11-17T02:03:55,068 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1731809034900.74108f0b014075d4993f0eee396e39e5. 2024-11-17T02:03:55,068 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,nnn,1731809034900.74108f0b014075d4993f0eee396e39e5. 2024-11-17T02:03:55,068 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 74108f0b014075d4993f0eee396e39e5: Waiting for close lock at 1731809035067Disabling compacts and flushes for region at 1731809035067Disabling writes for close at 1731809035068 (+1 ms)Writing region close event to WAL at 1731809035068Closed at 1731809035068 2024-11-17T02:03:55,068 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 8e1c1fc49350743a9cde50c433c8cb19, NAME => 'test,ppp,1731809034900.8e1c1fc49350743a9cde50c433c8cb19.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe 2024-11-17T02:03:55,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741850_1026 (size=45) 2024-11-17T02:03:55,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741850_1026 (size=45) 2024-11-17T02:03:55,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741850_1026 (size=45) 2024-11-17T02:03:55,073 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,mmm,1731809034900.663bac36df6c9873ad776bb3ad42409f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:55,073 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 663bac36df6c9873ad776bb3ad42409f, disabling compactions & flushes 2024-11-17T02:03:55,073 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,mmm,1731809034900.663bac36df6c9873ad776bb3ad42409f. 2024-11-17T02:03:55,074 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1731809034900.663bac36df6c9873ad776bb3ad42409f. 2024-11-17T02:03:55,074 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1731809034900.663bac36df6c9873ad776bb3ad42409f. after waiting 0 ms 2024-11-17T02:03:55,074 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1731809034900.663bac36df6c9873ad776bb3ad42409f. 2024-11-17T02:03:55,074 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,mmm,1731809034900.663bac36df6c9873ad776bb3ad42409f. 2024-11-17T02:03:55,074 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 663bac36df6c9873ad776bb3ad42409f: Waiting for close lock at 1731809035073Disabling compacts and flushes for region at 1731809035073Disabling writes for close at 1731809035074 (+1 ms)Writing region close event to WAL at 1731809035074Closed at 1731809035074 2024-11-17T02:03:55,074 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 899f31eacba6d2d0aedbee7d8bbb4c9d, NAME => 'test,qqq,1731809034900.899f31eacba6d2d0aedbee7d8bbb4c9d.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe 2024-11-17T02:03:55,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741852_1028 (size=45) 2024-11-17T02:03:55,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741852_1028 (size=45) 2024-11-17T02:03:55,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741852_1028 (size=45) 2024-11-17T02:03:55,083 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ooo,1731809034900.25a67f85380b4a3cd6890ab81d0b3a27.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:55,083 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 25a67f85380b4a3cd6890ab81d0b3a27, disabling compactions & flushes 2024-11-17T02:03:55,083 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ooo,1731809034900.25a67f85380b4a3cd6890ab81d0b3a27. 2024-11-17T02:03:55,083 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1731809034900.25a67f85380b4a3cd6890ab81d0b3a27. 2024-11-17T02:03:55,083 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1731809034900.25a67f85380b4a3cd6890ab81d0b3a27. after waiting 0 ms 2024-11-17T02:03:55,083 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1731809034900.25a67f85380b4a3cd6890ab81d0b3a27. 2024-11-17T02:03:55,083 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ooo,1731809034900.25a67f85380b4a3cd6890ab81d0b3a27. 2024-11-17T02:03:55,083 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 25a67f85380b4a3cd6890ab81d0b3a27: Waiting for close lock at 1731809035083Disabling compacts and flushes for region at 1731809035083Disabling writes for close at 1731809035083Writing region close event to WAL at 1731809035083Closed at 1731809035083 2024-11-17T02:03:55,084 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b7934444caa228b9e5d6d7b28b59a796, NAME => 'test,rrr,1731809034900.b7934444caa228b9e5d6d7b28b59a796.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe 2024-11-17T02:03:55,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741853_1029 (size=45) 2024-11-17T02:03:55,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741853_1029 (size=45) 2024-11-17T02:03:55,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741853_1029 (size=45) 2024-11-17T02:03:55,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741854_1030 (size=45) 2024-11-17T02:03:55,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741854_1030 (size=45) 2024-11-17T02:03:55,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741854_1030 (size=45) 2024-11-17T02:03:55,098 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,qqq,1731809034900.899f31eacba6d2d0aedbee7d8bbb4c9d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:55,099 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 899f31eacba6d2d0aedbee7d8bbb4c9d, disabling compactions & flushes 2024-11-17T02:03:55,099 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,qqq,1731809034900.899f31eacba6d2d0aedbee7d8bbb4c9d. 2024-11-17T02:03:55,099 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1731809034900.899f31eacba6d2d0aedbee7d8bbb4c9d. 2024-11-17T02:03:55,099 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1731809034900.899f31eacba6d2d0aedbee7d8bbb4c9d. after waiting 0 ms 2024-11-17T02:03:55,099 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1731809034900.899f31eacba6d2d0aedbee7d8bbb4c9d. 2024-11-17T02:03:55,099 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,qqq,1731809034900.899f31eacba6d2d0aedbee7d8bbb4c9d. 2024-11-17T02:03:55,099 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 899f31eacba6d2d0aedbee7d8bbb4c9d: Waiting for close lock at 1731809035099Disabling compacts and flushes for region at 1731809035099Disabling writes for close at 1731809035099Writing region close event to WAL at 1731809035099Closed at 1731809035099 2024-11-17T02:03:55,100 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => b6eba00b85aa16d5ccbfc8e3c35ff298, NAME => 'test,sss,1731809034900.b6eba00b85aa16d5ccbfc8e3c35ff298.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe 2024-11-17T02:03:55,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741855_1031 (size=45) 2024-11-17T02:03:55,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741855_1031 (size=45) 2024-11-17T02:03:55,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741855_1031 (size=45) 2024-11-17T02:03:55,104 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,rrr,1731809034900.b7934444caa228b9e5d6d7b28b59a796.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:55,104 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing b7934444caa228b9e5d6d7b28b59a796, disabling compactions & flushes 2024-11-17T02:03:55,104 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,rrr,1731809034900.b7934444caa228b9e5d6d7b28b59a796. 2024-11-17T02:03:55,104 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1731809034900.b7934444caa228b9e5d6d7b28b59a796. 2024-11-17T02:03:55,104 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1731809034900.b7934444caa228b9e5d6d7b28b59a796. after waiting 0 ms 2024-11-17T02:03:55,104 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1731809034900.b7934444caa228b9e5d6d7b28b59a796. 2024-11-17T02:03:55,104 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,rrr,1731809034900.b7934444caa228b9e5d6d7b28b59a796. 2024-11-17T02:03:55,105 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for b7934444caa228b9e5d6d7b28b59a796: Waiting for close lock at 1731809035104Disabling compacts and flushes for region at 1731809035104Disabling writes for close at 1731809035104Writing region close event to WAL at 1731809035104Closed at 1731809035104 2024-11-17T02:03:55,105 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4748cc4c5ad3838a3d0568cee797c169, NAME => 'test,ttt,1731809034900.4748cc4c5ad3838a3d0568cee797c169.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe 2024-11-17T02:03:55,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741856_1032 (size=45) 2024-11-17T02:03:55,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741856_1032 (size=45) 2024-11-17T02:03:55,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741856_1032 (size=45) 2024-11-17T02:03:55,112 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,sss,1731809034900.b6eba00b85aa16d5ccbfc8e3c35ff298.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:55,112 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing b6eba00b85aa16d5ccbfc8e3c35ff298, disabling compactions & flushes 2024-11-17T02:03:55,112 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,sss,1731809034900.b6eba00b85aa16d5ccbfc8e3c35ff298. 2024-11-17T02:03:55,112 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1731809034900.b6eba00b85aa16d5ccbfc8e3c35ff298. 2024-11-17T02:03:55,112 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1731809034900.b6eba00b85aa16d5ccbfc8e3c35ff298. after waiting 0 ms 2024-11-17T02:03:55,113 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1731809034900.b6eba00b85aa16d5ccbfc8e3c35ff298. 2024-11-17T02:03:55,113 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,sss,1731809034900.b6eba00b85aa16d5ccbfc8e3c35ff298. 2024-11-17T02:03:55,113 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for b6eba00b85aa16d5ccbfc8e3c35ff298: Waiting for close lock at 1731809035112Disabling compacts and flushes for region at 1731809035112Disabling writes for close at 1731809035112Writing region close event to WAL at 1731809035113 (+1 ms)Closed at 1731809035113 2024-11-17T02:03:55,113 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 77bd97fde6bd169f3e20553b45a65e2c, NAME => 'test,uuu,1731809034900.77bd97fde6bd169f3e20553b45a65e2c.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe 2024-11-17T02:03:55,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741857_1033 (size=45) 2024-11-17T02:03:55,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741857_1033 (size=45) 2024-11-17T02:03:55,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741857_1033 (size=45) 2024-11-17T02:03:55,121 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ttt,1731809034900.4748cc4c5ad3838a3d0568cee797c169.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:55,121 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 4748cc4c5ad3838a3d0568cee797c169, disabling compactions & flushes 2024-11-17T02:03:55,121 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ttt,1731809034900.4748cc4c5ad3838a3d0568cee797c169. 2024-11-17T02:03:55,121 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1731809034900.4748cc4c5ad3838a3d0568cee797c169. 2024-11-17T02:03:55,121 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1731809034900.4748cc4c5ad3838a3d0568cee797c169. after waiting 0 ms 2024-11-17T02:03:55,121 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1731809034900.4748cc4c5ad3838a3d0568cee797c169. 2024-11-17T02:03:55,121 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ttt,1731809034900.4748cc4c5ad3838a3d0568cee797c169. 2024-11-17T02:03:55,121 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4748cc4c5ad3838a3d0568cee797c169: Waiting for close lock at 1731809035121Disabling compacts and flushes for region at 1731809035121Disabling writes for close at 1731809035121Writing region close event to WAL at 1731809035121Closed at 1731809035121 2024-11-17T02:03:55,122 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2f4fe90ec03839f5d26acd9d02569e56, NAME => 'test,vvv,1731809034900.2f4fe90ec03839f5d26acd9d02569e56.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe 2024-11-17T02:03:55,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741858_1034 (size=45) 2024-11-17T02:03:55,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741858_1034 (size=45) 2024-11-17T02:03:55,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741858_1034 (size=45) 2024-11-17T02:03:55,129 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,uuu,1731809034900.77bd97fde6bd169f3e20553b45a65e2c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:55,129 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 77bd97fde6bd169f3e20553b45a65e2c, disabling compactions & flushes 2024-11-17T02:03:55,129 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,uuu,1731809034900.77bd97fde6bd169f3e20553b45a65e2c. 2024-11-17T02:03:55,130 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1731809034900.77bd97fde6bd169f3e20553b45a65e2c. 2024-11-17T02:03:55,130 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1731809034900.77bd97fde6bd169f3e20553b45a65e2c. after waiting 0 ms 2024-11-17T02:03:55,130 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1731809034900.77bd97fde6bd169f3e20553b45a65e2c. 2024-11-17T02:03:55,130 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,uuu,1731809034900.77bd97fde6bd169f3e20553b45a65e2c. 2024-11-17T02:03:55,130 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 77bd97fde6bd169f3e20553b45a65e2c: Waiting for close lock at 1731809035129Disabling compacts and flushes for region at 1731809035129Disabling writes for close at 1731809035130 (+1 ms)Writing region close event to WAL at 1731809035130Closed at 1731809035130 2024-11-17T02:03:55,131 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => db157dde5b6770afbf869f7733d95efb, NAME => 'test,www,1731809034900.db157dde5b6770afbf869f7733d95efb.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe 2024-11-17T02:03:55,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741859_1035 (size=45) 2024-11-17T02:03:55,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741859_1035 (size=45) 2024-11-17T02:03:55,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741859_1035 (size=45) 2024-11-17T02:03:55,143 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,vvv,1731809034900.2f4fe90ec03839f5d26acd9d02569e56.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:55,143 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 2f4fe90ec03839f5d26acd9d02569e56, disabling compactions & flushes 2024-11-17T02:03:55,143 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,vvv,1731809034900.2f4fe90ec03839f5d26acd9d02569e56. 2024-11-17T02:03:55,143 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1731809034900.2f4fe90ec03839f5d26acd9d02569e56. 2024-11-17T02:03:55,143 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1731809034900.2f4fe90ec03839f5d26acd9d02569e56. after waiting 0 ms 2024-11-17T02:03:55,143 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1731809034900.2f4fe90ec03839f5d26acd9d02569e56. 2024-11-17T02:03:55,143 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,vvv,1731809034900.2f4fe90ec03839f5d26acd9d02569e56. 2024-11-17T02:03:55,144 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2f4fe90ec03839f5d26acd9d02569e56: Waiting for close lock at 1731809035143Disabling compacts and flushes for region at 1731809035143Disabling writes for close at 1731809035143Writing region close event to WAL at 1731809035143Closed at 1731809035143 2024-11-17T02:03:55,144 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9d86e1fe125f78846dcaa5b9921c72db, NAME => 'test,xxx,1731809034900.9d86e1fe125f78846dcaa5b9921c72db.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe 2024-11-17T02:03:55,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741860_1036 (size=45) 2024-11-17T02:03:55,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741860_1036 (size=45) 2024-11-17T02:03:55,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741860_1036 (size=45) 2024-11-17T02:03:55,147 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,www,1731809034900.db157dde5b6770afbf869f7733d95efb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:55,147 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing db157dde5b6770afbf869f7733d95efb, disabling compactions & flushes 2024-11-17T02:03:55,147 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,www,1731809034900.db157dde5b6770afbf869f7733d95efb. 2024-11-17T02:03:55,147 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1731809034900.db157dde5b6770afbf869f7733d95efb. 2024-11-17T02:03:55,147 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1731809034900.db157dde5b6770afbf869f7733d95efb. after waiting 0 ms 2024-11-17T02:03:55,147 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1731809034900.db157dde5b6770afbf869f7733d95efb. 2024-11-17T02:03:55,147 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,www,1731809034900.db157dde5b6770afbf869f7733d95efb. 2024-11-17T02:03:55,147 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for db157dde5b6770afbf869f7733d95efb: Waiting for close lock at 1731809035147Disabling compacts and flushes for region at 1731809035147Disabling writes for close at 1731809035147Writing region close event to WAL at 1731809035147Closed at 1731809035147 2024-11-17T02:03:55,148 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 7eed07a820ab15d7af64462e2c1b70f7, NAME => 'test,yyy,1731809034900.7eed07a820ab15d7af64462e2c1b70f7.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe 2024-11-17T02:03:55,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741861_1037 (size=45) 2024-11-17T02:03:55,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741861_1037 (size=45) 2024-11-17T02:03:55,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741861_1037 (size=45) 2024-11-17T02:03:55,156 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,xxx,1731809034900.9d86e1fe125f78846dcaa5b9921c72db.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:55,157 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 9d86e1fe125f78846dcaa5b9921c72db, disabling compactions & flushes 2024-11-17T02:03:55,157 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,xxx,1731809034900.9d86e1fe125f78846dcaa5b9921c72db. 2024-11-17T02:03:55,157 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1731809034900.9d86e1fe125f78846dcaa5b9921c72db. 2024-11-17T02:03:55,157 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1731809034900.9d86e1fe125f78846dcaa5b9921c72db. after waiting 0 ms 2024-11-17T02:03:55,157 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1731809034900.9d86e1fe125f78846dcaa5b9921c72db. 2024-11-17T02:03:55,157 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,xxx,1731809034900.9d86e1fe125f78846dcaa5b9921c72db. 2024-11-17T02:03:55,157 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9d86e1fe125f78846dcaa5b9921c72db: Waiting for close lock at 1731809035157Disabling compacts and flushes for region at 1731809035157Disabling writes for close at 1731809035157Writing region close event to WAL at 1731809035157Closed at 1731809035157 2024-11-17T02:03:55,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741862_1038 (size=45) 2024-11-17T02:03:55,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741862_1038 (size=45) 2024-11-17T02:03:55,158 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 037b7a93730957425bc54697b3594bb2, NAME => 'test,zzz,1731809034900.037b7a93730957425bc54697b3594bb2.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe 2024-11-17T02:03:55,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741862_1038 (size=45) 2024-11-17T02:03:55,159 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,yyy,1731809034900.7eed07a820ab15d7af64462e2c1b70f7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:55,159 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 7eed07a820ab15d7af64462e2c1b70f7, disabling compactions & flushes 2024-11-17T02:03:55,159 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,yyy,1731809034900.7eed07a820ab15d7af64462e2c1b70f7. 2024-11-17T02:03:55,159 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1731809034900.7eed07a820ab15d7af64462e2c1b70f7. 2024-11-17T02:03:55,159 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1731809034900.7eed07a820ab15d7af64462e2c1b70f7. after waiting 0 ms 2024-11-17T02:03:55,159 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1731809034900.7eed07a820ab15d7af64462e2c1b70f7. 2024-11-17T02:03:55,159 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,yyy,1731809034900.7eed07a820ab15d7af64462e2c1b70f7. 2024-11-17T02:03:55,159 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 7eed07a820ab15d7af64462e2c1b70f7: Waiting for close lock at 1731809035159Disabling compacts and flushes for region at 1731809035159Disabling writes for close at 1731809035159Writing region close event to WAL at 1731809035159Closed at 1731809035159 2024-11-17T02:03:55,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741863_1039 (size=42) 2024-11-17T02:03:55,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741863_1039 (size=42) 2024-11-17T02:03:55,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741863_1039 (size=42) 2024-11-17T02:03:55,168 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,zzz,1731809034900.037b7a93730957425bc54697b3594bb2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:55,168 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 037b7a93730957425bc54697b3594bb2, disabling compactions & flushes 2024-11-17T02:03:55,168 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,zzz,1731809034900.037b7a93730957425bc54697b3594bb2. 2024-11-17T02:03:55,168 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1731809034900.037b7a93730957425bc54697b3594bb2. 2024-11-17T02:03:55,168 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1731809034900.037b7a93730957425bc54697b3594bb2. after waiting 0 ms 2024-11-17T02:03:55,168 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1731809034900.037b7a93730957425bc54697b3594bb2. 2024-11-17T02:03:55,168 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,zzz,1731809034900.037b7a93730957425bc54697b3594bb2. 2024-11-17T02:03:55,168 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 037b7a93730957425bc54697b3594bb2: Waiting for close lock at 1731809035168Disabling compacts and flushes for region at 1731809035168Disabling writes for close at 1731809035168Writing region close event to WAL at 1731809035168Closed at 1731809035168 2024-11-17T02:03:55,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T02:03:55,496 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ppp,1731809034900.8e1c1fc49350743a9cde50c433c8cb19.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:55,496 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 8e1c1fc49350743a9cde50c433c8cb19, disabling compactions & flushes 2024-11-17T02:03:55,496 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ppp,1731809034900.8e1c1fc49350743a9cde50c433c8cb19. 2024-11-17T02:03:55,496 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1731809034900.8e1c1fc49350743a9cde50c433c8cb19. 2024-11-17T02:03:55,496 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1731809034900.8e1c1fc49350743a9cde50c433c8cb19. after waiting 0 ms 2024-11-17T02:03:55,496 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1731809034900.8e1c1fc49350743a9cde50c433c8cb19. 2024-11-17T02:03:55,497 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ppp,1731809034900.8e1c1fc49350743a9cde50c433c8cb19. 2024-11-17T02:03:55,497 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 8e1c1fc49350743a9cde50c433c8cb19: Waiting for close lock at 1731809035496Disabling compacts and flushes for region at 1731809035496Disabling writes for close at 1731809035496Writing region close event to WAL at 1731809035496Closed at 1731809035497 (+1 ms) 2024-11-17T02:03:55,503 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T02:03:55,511 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1731809034900.a17ac9e270344fa2752f51f8196693b2.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731809035503"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731809035503"}]},"ts":"1731809035503"} 2024-11-17T02:03:55,512 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1731809034900.81f322eb57dcb42c3b4269a738bafd6d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731809035503"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731809035503"}]},"ts":"1731809035503"} 2024-11-17T02:03:55,512 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1731809034900.de9924be32211cddceef43e5b4866dc8.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731809035503"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731809035503"}]},"ts":"1731809035503"} 2024-11-17T02:03:55,512 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1731809034900.15c64fab21a3ca03ec0a7afabc19cdc9.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731809035503"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731809035503"}]},"ts":"1731809035503"} 2024-11-17T02:03:55,512 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1731809034900.8356c0fdb856c376b8678f2c2597565a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731809035503"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731809035503"}]},"ts":"1731809035503"} 2024-11-17T02:03:55,513 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1731809034900.112f5bbe15f72793585db9ba98950976.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731809035503"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731809035503"}]},"ts":"1731809035503"} 2024-11-17T02:03:55,513 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1731809034900.6fd33da09f9d938215ac0e7127ad70d5.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731809035503"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731809035503"}]},"ts":"1731809035503"} 2024-11-17T02:03:55,513 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1731809034900.7517bce9e04879764302cccd45af20fb.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731809035503"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731809035503"}]},"ts":"1731809035503"} 2024-11-17T02:03:55,513 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1731809034900.03b7adb3afa9bebd86dc1da38125b982.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731809035503"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731809035503"}]},"ts":"1731809035503"} 2024-11-17T02:03:55,513 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1731809034900.ac10d67ff6c3f527cc0861e738032bdb.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731809035503"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731809035503"}]},"ts":"1731809035503"} 2024-11-17T02:03:55,514 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1731809034900.05534a4a027a1fdc90227b7a2e681933.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731809035503"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731809035503"}]},"ts":"1731809035503"} 2024-11-17T02:03:55,514 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1731809034900.1056118c93716b24d18f749826cf316a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731809035503"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731809035503"}]},"ts":"1731809035503"} 2024-11-17T02:03:55,514 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1731809034900.74108f0b014075d4993f0eee396e39e5.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731809035503"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731809035503"}]},"ts":"1731809035503"} 2024-11-17T02:03:55,514 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1731809034900.663bac36df6c9873ad776bb3ad42409f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731809035503"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731809035503"}]},"ts":"1731809035503"} 2024-11-17T02:03:55,514 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1731809034900.25a67f85380b4a3cd6890ab81d0b3a27.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731809035503"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731809035503"}]},"ts":"1731809035503"} 2024-11-17T02:03:55,515 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1731809034900.899f31eacba6d2d0aedbee7d8bbb4c9d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731809035503"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731809035503"}]},"ts":"1731809035503"} 2024-11-17T02:03:55,515 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1731809034900.b7934444caa228b9e5d6d7b28b59a796.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731809035503"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731809035503"}]},"ts":"1731809035503"} 2024-11-17T02:03:55,515 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1731809034900.b6eba00b85aa16d5ccbfc8e3c35ff298.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731809035503"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731809035503"}]},"ts":"1731809035503"} 2024-11-17T02:03:55,515 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1731809034900.4748cc4c5ad3838a3d0568cee797c169.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731809035503"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731809035503"}]},"ts":"1731809035503"} 2024-11-17T02:03:55,515 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1731809034900.77bd97fde6bd169f3e20553b45a65e2c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731809035503"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731809035503"}]},"ts":"1731809035503"} 2024-11-17T02:03:55,516 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1731809034900.2f4fe90ec03839f5d26acd9d02569e56.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731809035503"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731809035503"}]},"ts":"1731809035503"} 2024-11-17T02:03:55,516 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1731809034900.db157dde5b6770afbf869f7733d95efb.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731809035503"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731809035503"}]},"ts":"1731809035503"} 2024-11-17T02:03:55,516 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1731809034900.9d86e1fe125f78846dcaa5b9921c72db.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731809035503"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731809035503"}]},"ts":"1731809035503"} 2024-11-17T02:03:55,516 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1731809034900.7eed07a820ab15d7af64462e2c1b70f7.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731809035503"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731809035503"}]},"ts":"1731809035503"} 2024-11-17T02:03:55,516 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1731809034900.037b7a93730957425bc54697b3594bb2.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731809035503"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731809035503"}]},"ts":"1731809035503"} 2024-11-17T02:03:55,516 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1731809034900.8e1c1fc49350743a9cde50c433c8cb19.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731809035503"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731809035503"}]},"ts":"1731809035503"} 2024-11-17T02:03:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T02:03:55,560 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-17T02:03:55,563 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T02:03:55,565 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731809035563"}]},"ts":"1731809035563"} 2024-11-17T02:03:55,569 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-11-17T02:03:55,570 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {c6c0d7a0a7c0=0} racks are {/default-rack=0} 2024-11-17T02:03:55,588 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-17T02:03:55,588 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-17T02:03:55,588 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-17T02:03:55,588 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-17T02:03:55,588 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-17T02:03:55,588 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-17T02:03:55,588 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-17T02:03:55,589 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-17T02:03:55,589 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-17T02:03:55,589 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-17T02:03:55,592 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a17ac9e270344fa2752f51f8196693b2, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=81f322eb57dcb42c3b4269a738bafd6d, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=15c64fab21a3ca03ec0a7afabc19cdc9, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8356c0fdb856c376b8678f2c2597565a, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7517bce9e04879764302cccd45af20fb, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6fd33da09f9d938215ac0e7127ad70d5, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=05534a4a027a1fdc90227b7a2e681933, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ac10d67ff6c3f527cc0861e738032bdb, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=663bac36df6c9873ad776bb3ad42409f, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=74108f0b014075d4993f0eee396e39e5, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8e1c1fc49350743a9cde50c433c8cb19, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=899f31eacba6d2d0aedbee7d8bbb4c9d, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b7934444caa228b9e5d6d7b28b59a796, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4748cc4c5ad3838a3d0568cee797c169, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=77bd97fde6bd169f3e20553b45a65e2c, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=db157dde5b6770afbf869f7733d95efb, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9d86e1fe125f78846dcaa5b9921c72db, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=037b7a93730957425bc54697b3594bb2, ASSIGN}] 2024-11-17T02:03:55,603 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=037b7a93730957425bc54697b3594bb2, ASSIGN 2024-11-17T02:03:55,604 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=037b7a93730957425bc54697b3594bb2, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,45037,1731809032394; forceNewPlan=false, retain=false 2024-11-17T02:03:55,605 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN 2024-11-17T02:03:55,607 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,34033,1731809032455; forceNewPlan=false, retain=false 2024-11-17T02:03:55,608 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9d86e1fe125f78846dcaa5b9921c72db, ASSIGN 2024-11-17T02:03:55,609 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9d86e1fe125f78846dcaa5b9921c72db, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,33127,1731809032263; forceNewPlan=false, retain=false 2024-11-17T02:03:55,610 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=db157dde5b6770afbf869f7733d95efb, ASSIGN 2024-11-17T02:03:55,612 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=db157dde5b6770afbf869f7733d95efb, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,45037,1731809032394; forceNewPlan=false, retain=false 2024-11-17T02:03:55,613 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN 2024-11-17T02:03:55,614 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,34033,1731809032455; forceNewPlan=false, retain=false 2024-11-17T02:03:55,616 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=77bd97fde6bd169f3e20553b45a65e2c, ASSIGN 2024-11-17T02:03:55,617 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=77bd97fde6bd169f3e20553b45a65e2c, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,33127,1731809032263; forceNewPlan=false, retain=false 2024-11-17T02:03:55,618 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4748cc4c5ad3838a3d0568cee797c169, ASSIGN 2024-11-17T02:03:55,620 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4748cc4c5ad3838a3d0568cee797c169, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,45037,1731809032394; forceNewPlan=false, retain=false 2024-11-17T02:03:55,621 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN 2024-11-17T02:03:55,622 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,34033,1731809032455; forceNewPlan=false, retain=false 2024-11-17T02:03:55,624 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b7934444caa228b9e5d6d7b28b59a796, ASSIGN 2024-11-17T02:03:55,625 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b7934444caa228b9e5d6d7b28b59a796, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,33127,1731809032263; forceNewPlan=false, retain=false 2024-11-17T02:03:55,626 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=899f31eacba6d2d0aedbee7d8bbb4c9d, ASSIGN 2024-11-17T02:03:55,628 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=899f31eacba6d2d0aedbee7d8bbb4c9d, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,45037,1731809032394; forceNewPlan=false, retain=false 2024-11-17T02:03:55,629 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8e1c1fc49350743a9cde50c433c8cb19, ASSIGN 2024-11-17T02:03:55,630 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8e1c1fc49350743a9cde50c433c8cb19, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,33127,1731809032263; forceNewPlan=false, retain=false 2024-11-17T02:03:55,631 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN 2024-11-17T02:03:55,633 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,34033,1731809032455; forceNewPlan=false, retain=false 2024-11-17T02:03:55,634 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=74108f0b014075d4993f0eee396e39e5, ASSIGN 2024-11-17T02:03:55,635 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=74108f0b014075d4993f0eee396e39e5, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,45037,1731809032394; forceNewPlan=false, retain=false 2024-11-17T02:03:55,637 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=663bac36df6c9873ad776bb3ad42409f, ASSIGN 2024-11-17T02:03:55,638 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=663bac36df6c9873ad776bb3ad42409f, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,33127,1731809032263; forceNewPlan=false, retain=false 2024-11-17T02:03:55,639 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN 2024-11-17T02:03:55,641 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,34033,1731809032455; forceNewPlan=false, retain=false 2024-11-17T02:03:55,642 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ac10d67ff6c3f527cc0861e738032bdb, ASSIGN 2024-11-17T02:03:55,643 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ac10d67ff6c3f527cc0861e738032bdb, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,45037,1731809032394; forceNewPlan=false, retain=false 2024-11-17T02:03:55,645 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=05534a4a027a1fdc90227b7a2e681933, ASSIGN 2024-11-17T02:03:55,646 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=05534a4a027a1fdc90227b7a2e681933, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,33127,1731809032263; forceNewPlan=false, retain=false 2024-11-17T02:03:55,647 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN 2024-11-17T02:03:55,649 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,34033,1731809032455; forceNewPlan=false, retain=false 2024-11-17T02:03:55,650 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6fd33da09f9d938215ac0e7127ad70d5, ASSIGN 2024-11-17T02:03:55,651 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6fd33da09f9d938215ac0e7127ad70d5, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,45037,1731809032394; forceNewPlan=false, retain=false 2024-11-17T02:03:55,652 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7517bce9e04879764302cccd45af20fb, ASSIGN 2024-11-17T02:03:55,653 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7517bce9e04879764302cccd45af20fb, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,33127,1731809032263; forceNewPlan=false, retain=false 2024-11-17T02:03:55,654 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN 2024-11-17T02:03:55,656 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,34033,1731809032455; forceNewPlan=false, retain=false 2024-11-17T02:03:55,657 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8356c0fdb856c376b8678f2c2597565a, ASSIGN 2024-11-17T02:03:55,659 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8356c0fdb856c376b8678f2c2597565a, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,33127,1731809032263; forceNewPlan=false, retain=false 2024-11-17T02:03:55,661 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=15c64fab21a3ca03ec0a7afabc19cdc9, ASSIGN 2024-11-17T02:03:55,662 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=15c64fab21a3ca03ec0a7afabc19cdc9, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,45037,1731809032394; forceNewPlan=false, retain=false 2024-11-17T02:03:55,664 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=81f322eb57dcb42c3b4269a738bafd6d, ASSIGN 2024-11-17T02:03:55,665 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=81f322eb57dcb42c3b4269a738bafd6d, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,33127,1731809032263; forceNewPlan=false, retain=false 2024-11-17T02:03:55,667 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a17ac9e270344fa2752f51f8196693b2, ASSIGN 2024-11-17T02:03:55,668 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a17ac9e270344fa2752f51f8196693b2, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,45037,1731809032394; forceNewPlan=false, retain=false 2024-11-17T02:03:55,670 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN 2024-11-17T02:03:55,671 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,34033,1731809032455; forceNewPlan=false, retain=false 2024-11-17T02:03:55,760 INFO [c6c0d7a0a7c0:32803 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-17T02:03:55,762 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=037b7a93730957425bc54697b3594bb2, regionState=OPENING, regionLocation=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:55,763 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=db157dde5b6770afbf869f7733d95efb, regionState=OPENING, regionLocation=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:55,763 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=4748cc4c5ad3838a3d0568cee797c169, regionState=OPENING, regionLocation=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:55,764 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=899f31eacba6d2d0aedbee7d8bbb4c9d, regionState=OPENING, regionLocation=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:55,765 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=74108f0b014075d4993f0eee396e39e5, regionState=OPENING, regionLocation=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:55,766 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=ac10d67ff6c3f527cc0861e738032bdb, regionState=OPENING, regionLocation=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:55,766 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=6fd33da09f9d938215ac0e7127ad70d5, regionState=OPENING, regionLocation=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:55,766 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=037b7a93730957425bc54697b3594bb2, ASSIGN because future has completed 2024-11-17T02:03:55,767 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=15c64fab21a3ca03ec0a7afabc19cdc9, regionState=OPENING, regionLocation=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:55,768 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 037b7a93730957425bc54697b3594bb2, server=c6c0d7a0a7c0,45037,1731809032394}] 2024-11-17T02:03:55,769 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=db157dde5b6770afbf869f7733d95efb, ASSIGN because future has completed 2024-11-17T02:03:55,771 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4748cc4c5ad3838a3d0568cee797c169, ASSIGN because future has completed 2024-11-17T02:03:55,773 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=899f31eacba6d2d0aedbee7d8bbb4c9d, ASSIGN because future has completed 2024-11-17T02:03:55,775 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=74108f0b014075d4993f0eee396e39e5, ASSIGN because future has completed 2024-11-17T02:03:55,776 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 74108f0b014075d4993f0eee396e39e5, server=c6c0d7a0a7c0,45037,1731809032394}] 2024-11-17T02:03:55,777 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ac10d67ff6c3f527cc0861e738032bdb, ASSIGN because future has completed 2024-11-17T02:03:55,779 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6fd33da09f9d938215ac0e7127ad70d5, ASSIGN because future has completed 2024-11-17T02:03:55,781 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=15c64fab21a3ca03ec0a7afabc19cdc9, ASSIGN because future has completed 2024-11-17T02:03:55,783 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 15c64fab21a3ca03ec0a7afabc19cdc9, server=c6c0d7a0a7c0,45037,1731809032394}] 2024-11-17T02:03:55,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6fd33da09f9d938215ac0e7127ad70d5, server=c6c0d7a0a7c0,45037,1731809032394}] 2024-11-17T02:03:55,794 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure ac10d67ff6c3f527cc0861e738032bdb, server=c6c0d7a0a7c0,45037,1731809032394}] 2024-11-17T02:03:55,799 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 899f31eacba6d2d0aedbee7d8bbb4c9d, server=c6c0d7a0a7c0,45037,1731809032394}] 2024-11-17T02:03:55,805 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4748cc4c5ad3838a3d0568cee797c169, server=c6c0d7a0a7c0,45037,1731809032394}] 2024-11-17T02:03:55,810 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure db157dde5b6770afbf869f7733d95efb, server=c6c0d7a0a7c0,45037,1731809032394}] 2024-11-17T02:03:55,814 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=a17ac9e270344fa2752f51f8196693b2, regionState=OPENING, regionLocation=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:55,815 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=7eed07a820ab15d7af64462e2c1b70f7, regionState=OPENING, regionLocation=c6c0d7a0a7c0,34033,1731809032455 2024-11-17T02:03:55,816 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=2f4fe90ec03839f5d26acd9d02569e56, regionState=OPENING, regionLocation=c6c0d7a0a7c0,34033,1731809032455 2024-11-17T02:03:55,816 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=b6eba00b85aa16d5ccbfc8e3c35ff298, regionState=OPENING, regionLocation=c6c0d7a0a7c0,34033,1731809032455 2024-11-17T02:03:55,817 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=25a67f85380b4a3cd6890ab81d0b3a27, regionState=OPENING, regionLocation=c6c0d7a0a7c0,34033,1731809032455 2024-11-17T02:03:55,818 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1056118c93716b24d18f749826cf316a, regionState=OPENING, regionLocation=c6c0d7a0a7c0,34033,1731809032455 2024-11-17T02:03:55,818 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=03b7adb3afa9bebd86dc1da38125b982, regionState=OPENING, regionLocation=c6c0d7a0a7c0,34033,1731809032455 2024-11-17T02:03:55,819 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=112f5bbe15f72793585db9ba98950976, regionState=OPENING, regionLocation=c6c0d7a0a7c0,34033,1731809032455 2024-11-17T02:03:55,820 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=de9924be32211cddceef43e5b4866dc8, regionState=OPENING, regionLocation=c6c0d7a0a7c0,34033,1731809032455 2024-11-17T02:03:55,820 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=9d86e1fe125f78846dcaa5b9921c72db, regionState=OPENING, regionLocation=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:55,821 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=77bd97fde6bd169f3e20553b45a65e2c, regionState=OPENING, regionLocation=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:55,822 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=b7934444caa228b9e5d6d7b28b59a796, regionState=OPENING, regionLocation=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:55,823 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=8e1c1fc49350743a9cde50c433c8cb19, regionState=OPENING, regionLocation=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:55,823 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a17ac9e270344fa2752f51f8196693b2, ASSIGN because future has completed 2024-11-17T02:03:55,823 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure a17ac9e270344fa2752f51f8196693b2, server=c6c0d7a0a7c0,45037,1731809032394}] 2024-11-17T02:03:55,826 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN because future has completed 2024-11-17T02:03:55,826 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7eed07a820ab15d7af64462e2c1b70f7, server=c6c0d7a0a7c0,34033,1731809032455}] 2024-11-17T02:03:55,830 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN because future has completed 2024-11-17T02:03:55,832 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN because future has completed 2024-11-17T02:03:55,834 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN because future has completed 2024-11-17T02:03:55,834 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure b6eba00b85aa16d5ccbfc8e3c35ff298, server=c6c0d7a0a7c0,34033,1731809032455}] 2024-11-17T02:03:55,835 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN because future has completed 2024-11-17T02:03:55,837 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN because future has completed 2024-11-17T02:03:55,840 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN because future has completed 2024-11-17T02:03:55,841 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 112f5bbe15f72793585db9ba98950976, server=c6c0d7a0a7c0,34033,1731809032455}] 2024-11-17T02:03:55,841 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN because future has completed 2024-11-17T02:03:55,843 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9d86e1fe125f78846dcaa5b9921c72db, ASSIGN because future has completed 2024-11-17T02:03:55,845 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=77bd97fde6bd169f3e20553b45a65e2c, ASSIGN because future has completed 2024-11-17T02:03:55,847 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b7934444caa228b9e5d6d7b28b59a796, ASSIGN because future has completed 2024-11-17T02:03:55,847 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 77bd97fde6bd169f3e20553b45a65e2c, server=c6c0d7a0a7c0,33127,1731809032263}] 2024-11-17T02:03:55,849 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8e1c1fc49350743a9cde50c433c8cb19, ASSIGN because future has completed 2024-11-17T02:03:55,852 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8e1c1fc49350743a9cde50c433c8cb19, server=c6c0d7a0a7c0,33127,1731809032263}] 2024-11-17T02:03:55,858 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7934444caa228b9e5d6d7b28b59a796, server=c6c0d7a0a7c0,33127,1731809032263}] 2024-11-17T02:03:55,864 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d86e1fe125f78846dcaa5b9921c72db, server=c6c0d7a0a7c0,33127,1731809032263}] 2024-11-17T02:03:55,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure de9924be32211cddceef43e5b4866dc8, server=c6c0d7a0a7c0,34033,1731809032455}] 2024-11-17T02:03:55,875 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,34033,1731809032455}] 2024-11-17T02:03:55,880 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1056118c93716b24d18f749826cf316a, server=c6c0d7a0a7c0,34033,1731809032455}] 2024-11-17T02:03:55,885 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 25a67f85380b4a3cd6890ab81d0b3a27, server=c6c0d7a0a7c0,34033,1731809032455}] 2024-11-17T02:03:55,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2f4fe90ec03839f5d26acd9d02569e56, server=c6c0d7a0a7c0,34033,1731809032455}] 2024-11-17T02:03:55,896 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=663bac36df6c9873ad776bb3ad42409f, regionState=OPENING, regionLocation=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:55,896 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=05534a4a027a1fdc90227b7a2e681933, regionState=OPENING, regionLocation=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:55,897 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=7517bce9e04879764302cccd45af20fb, regionState=OPENING, regionLocation=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:55,897 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=8356c0fdb856c376b8678f2c2597565a, regionState=OPENING, regionLocation=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:55,898 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=81f322eb57dcb42c3b4269a738bafd6d, regionState=OPENING, regionLocation=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:55,899 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=663bac36df6c9873ad776bb3ad42409f, ASSIGN because future has completed 2024-11-17T02:03:55,899 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 663bac36df6c9873ad776bb3ad42409f, server=c6c0d7a0a7c0,33127,1731809032263}] 2024-11-17T02:03:55,900 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=05534a4a027a1fdc90227b7a2e681933, ASSIGN because future has completed 2024-11-17T02:03:55,902 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7517bce9e04879764302cccd45af20fb, ASSIGN because future has completed 2024-11-17T02:03:55,903 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8356c0fdb856c376b8678f2c2597565a, ASSIGN because future has completed 2024-11-17T02:03:55,904 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8356c0fdb856c376b8678f2c2597565a, server=c6c0d7a0a7c0,33127,1731809032263}] 2024-11-17T02:03:55,905 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=81f322eb57dcb42c3b4269a738bafd6d, ASSIGN because future has completed 2024-11-17T02:03:55,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81f322eb57dcb42c3b4269a738bafd6d, server=c6c0d7a0a7c0,33127,1731809032263}] 2024-11-17T02:03:55,912 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7517bce9e04879764302cccd45af20fb, server=c6c0d7a0a7c0,33127,1731809032263}] 2024-11-17T02:03:55,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 05534a4a027a1fdc90227b7a2e681933, server=c6c0d7a0a7c0,33127,1731809032263}] 2024-11-17T02:03:55,923 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-17T02:03:55,925 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47205, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-17T02:03:55,932 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,bbb,1731809034900.a17ac9e270344fa2752f51f8196693b2. 2024-11-17T02:03:55,933 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => a17ac9e270344fa2752f51f8196693b2, NAME => 'test,bbb,1731809034900.a17ac9e270344fa2752f51f8196693b2.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-17T02:03:55,933 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a17ac9e270344fa2752f51f8196693b2 2024-11-17T02:03:55,933 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,bbb,1731809034900.a17ac9e270344fa2752f51f8196693b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:55,934 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for a17ac9e270344fa2752f51f8196693b2 2024-11-17T02:03:55,934 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for a17ac9e270344fa2752f51f8196693b2 2024-11-17T02:03:55,936 INFO [StoreOpener-a17ac9e270344fa2752f51f8196693b2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a17ac9e270344fa2752f51f8196693b2 2024-11-17T02:03:55,939 INFO [StoreOpener-a17ac9e270344fa2752f51f8196693b2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a17ac9e270344fa2752f51f8196693b2 columnFamilyName family 2024-11-17T02:03:55,939 DEBUG [StoreOpener-a17ac9e270344fa2752f51f8196693b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:55,939 INFO [StoreOpener-a17ac9e270344fa2752f51f8196693b2-1 {}] regionserver.HStore(327): Store=a17ac9e270344fa2752f51f8196693b2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T02:03:55,940 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for a17ac9e270344fa2752f51f8196693b2 2024-11-17T02:03:55,941 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/a17ac9e270344fa2752f51f8196693b2 2024-11-17T02:03:55,941 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/a17ac9e270344fa2752f51f8196693b2 2024-11-17T02:03:55,942 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for a17ac9e270344fa2752f51f8196693b2 2024-11-17T02:03:55,942 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for a17ac9e270344fa2752f51f8196693b2 2024-11-17T02:03:55,944 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for a17ac9e270344fa2752f51f8196693b2 2024-11-17T02:03:55,946 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/a17ac9e270344fa2752f51f8196693b2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T02:03:55,947 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened a17ac9e270344fa2752f51f8196693b2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69705843, jitterRate=0.03869800269603729}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T02:03:55,947 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a17ac9e270344fa2752f51f8196693b2 2024-11-17T02:03:55,948 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for a17ac9e270344fa2752f51f8196693b2: Running coprocessor pre-open hook at 1731809035934Writing region info on filesystem at 1731809035934Initializing all the Stores at 1731809035936 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731809035936Cleaning up temporary data from old regions at 1731809035942 (+6 ms)Running coprocessor post-open hooks at 1731809035947 (+5 ms)Region opened successfully at 1731809035948 (+1 ms) 2024-11-17T02:03:55,950 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1731809034900.a17ac9e270344fa2752f51f8196693b2., pid=39, masterSystemTime=1731809035923 2024-11-17T02:03:55,953 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1731809034900.a17ac9e270344fa2752f51f8196693b2. 2024-11-17T02:03:55,953 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,bbb,1731809034900.a17ac9e270344fa2752f51f8196693b2. 2024-11-17T02:03:55,953 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,qqq,1731809034900.899f31eacba6d2d0aedbee7d8bbb4c9d. 2024-11-17T02:03:55,954 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 899f31eacba6d2d0aedbee7d8bbb4c9d, NAME => 'test,qqq,1731809034900.899f31eacba6d2d0aedbee7d8bbb4c9d.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-17T02:03:55,954 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=a17ac9e270344fa2752f51f8196693b2, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:55,954 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 899f31eacba6d2d0aedbee7d8bbb4c9d 2024-11-17T02:03:55,954 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,qqq,1731809034900.899f31eacba6d2d0aedbee7d8bbb4c9d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:55,954 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 899f31eacba6d2d0aedbee7d8bbb4c9d 2024-11-17T02:03:55,954 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 899f31eacba6d2d0aedbee7d8bbb4c9d 2024-11-17T02:03:55,957 INFO [StoreOpener-899f31eacba6d2d0aedbee7d8bbb4c9d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 899f31eacba6d2d0aedbee7d8bbb4c9d 2024-11-17T02:03:55,957 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure a17ac9e270344fa2752f51f8196693b2, server=c6c0d7a0a7c0,45037,1731809032394 because future has completed 2024-11-17T02:03:55,959 INFO [StoreOpener-899f31eacba6d2d0aedbee7d8bbb4c9d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 899f31eacba6d2d0aedbee7d8bbb4c9d columnFamilyName family 2024-11-17T02:03:55,960 DEBUG [StoreOpener-899f31eacba6d2d0aedbee7d8bbb4c9d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:55,961 INFO [StoreOpener-899f31eacba6d2d0aedbee7d8bbb4c9d-1 {}] regionserver.HStore(327): Store=899f31eacba6d2d0aedbee7d8bbb4c9d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T02:03:55,961 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 899f31eacba6d2d0aedbee7d8bbb4c9d 2024-11-17T02:03:55,962 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=6 2024-11-17T02:03:55,962 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure a17ac9e270344fa2752f51f8196693b2, server=c6c0d7a0a7c0,45037,1731809032394 in 136 msec 2024-11-17T02:03:55,963 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/899f31eacba6d2d0aedbee7d8bbb4c9d 2024-11-17T02:03:55,963 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/899f31eacba6d2d0aedbee7d8bbb4c9d 2024-11-17T02:03:55,964 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 899f31eacba6d2d0aedbee7d8bbb4c9d 2024-11-17T02:03:55,964 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 899f31eacba6d2d0aedbee7d8bbb4c9d 2024-11-17T02:03:55,965 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a17ac9e270344fa2752f51f8196693b2, ASSIGN in 370 msec 2024-11-17T02:03:55,967 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 899f31eacba6d2d0aedbee7d8bbb4c9d 2024-11-17T02:03:55,970 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/899f31eacba6d2d0aedbee7d8bbb4c9d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T02:03:55,971 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 899f31eacba6d2d0aedbee7d8bbb4c9d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61669631, jitterRate=-0.08105088770389557}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T02:03:55,971 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 899f31eacba6d2d0aedbee7d8bbb4c9d 2024-11-17T02:03:55,971 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 899f31eacba6d2d0aedbee7d8bbb4c9d: Running coprocessor pre-open hook at 1731809035955Writing region info on filesystem at 1731809035955Initializing all the Stores at 1731809035956 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731809035957 (+1 ms)Cleaning up temporary data from old regions at 1731809035964 (+7 ms)Running coprocessor post-open hooks at 1731809035971 (+7 ms)Region opened successfully at 1731809035971 2024-11-17T02:03:55,973 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1731809034900.899f31eacba6d2d0aedbee7d8bbb4c9d., pid=36, masterSystemTime=1731809035923 2024-11-17T02:03:55,975 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1731809034900.899f31eacba6d2d0aedbee7d8bbb4c9d. 2024-11-17T02:03:55,975 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,qqq,1731809034900.899f31eacba6d2d0aedbee7d8bbb4c9d. 2024-11-17T02:03:55,976 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,ddd,1731809034900.15c64fab21a3ca03ec0a7afabc19cdc9. 2024-11-17T02:03:55,976 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 15c64fab21a3ca03ec0a7afabc19cdc9, NAME => 'test,ddd,1731809034900.15c64fab21a3ca03ec0a7afabc19cdc9.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-17T02:03:55,976 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 15c64fab21a3ca03ec0a7afabc19cdc9 2024-11-17T02:03:55,976 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=899f31eacba6d2d0aedbee7d8bbb4c9d, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:55,976 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,ddd,1731809034900.15c64fab21a3ca03ec0a7afabc19cdc9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:55,976 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 15c64fab21a3ca03ec0a7afabc19cdc9 2024-11-17T02:03:55,976 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 15c64fab21a3ca03ec0a7afabc19cdc9 2024-11-17T02:03:55,979 INFO [StoreOpener-15c64fab21a3ca03ec0a7afabc19cdc9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 15c64fab21a3ca03ec0a7afabc19cdc9 2024-11-17T02:03:55,979 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 899f31eacba6d2d0aedbee7d8bbb4c9d, server=c6c0d7a0a7c0,45037,1731809032394 because future has completed 2024-11-17T02:03:55,981 INFO [StoreOpener-15c64fab21a3ca03ec0a7afabc19cdc9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15c64fab21a3ca03ec0a7afabc19cdc9 columnFamilyName family 2024-11-17T02:03:55,981 DEBUG [StoreOpener-15c64fab21a3ca03ec0a7afabc19cdc9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:55,982 INFO [StoreOpener-15c64fab21a3ca03ec0a7afabc19cdc9-1 {}] regionserver.HStore(327): Store=15c64fab21a3ca03ec0a7afabc19cdc9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T02:03:55,982 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 15c64fab21a3ca03ec0a7afabc19cdc9 2024-11-17T02:03:55,983 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/15c64fab21a3ca03ec0a7afabc19cdc9 2024-11-17T02:03:55,984 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=21 2024-11-17T02:03:55,984 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 899f31eacba6d2d0aedbee7d8bbb4c9d, server=c6c0d7a0a7c0,45037,1731809032394 in 181 msec 2024-11-17T02:03:55,984 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/15c64fab21a3ca03ec0a7afabc19cdc9 2024-11-17T02:03:55,984 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 15c64fab21a3ca03ec0a7afabc19cdc9 2024-11-17T02:03:55,984 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 15c64fab21a3ca03ec0a7afabc19cdc9 2024-11-17T02:03:55,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=899f31eacba6d2d0aedbee7d8bbb4c9d, ASSIGN in 392 msec 2024-11-17T02:03:55,988 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 15c64fab21a3ca03ec0a7afabc19cdc9 2024-11-17T02:03:55,988 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,sss,1731809034900.b6eba00b85aa16d5ccbfc8e3c35ff298. 2024-11-17T02:03:55,988 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => b6eba00b85aa16d5ccbfc8e3c35ff298, NAME => 'test,sss,1731809034900.b6eba00b85aa16d5ccbfc8e3c35ff298.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-17T02:03:55,989 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b6eba00b85aa16d5ccbfc8e3c35ff298 2024-11-17T02:03:55,989 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,sss,1731809034900.b6eba00b85aa16d5ccbfc8e3c35ff298.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:55,989 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for b6eba00b85aa16d5ccbfc8e3c35ff298 2024-11-17T02:03:55,989 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for b6eba00b85aa16d5ccbfc8e3c35ff298 2024-11-17T02:03:55,991 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/15c64fab21a3ca03ec0a7afabc19cdc9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T02:03:55,991 INFO [StoreOpener-b6eba00b85aa16d5ccbfc8e3c35ff298-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b6eba00b85aa16d5ccbfc8e3c35ff298 2024-11-17T02:03:55,992 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 15c64fab21a3ca03ec0a7afabc19cdc9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72250023, jitterRate=0.07660923898220062}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T02:03:55,992 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 15c64fab21a3ca03ec0a7afabc19cdc9 2024-11-17T02:03:55,992 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 15c64fab21a3ca03ec0a7afabc19cdc9: Running coprocessor pre-open hook at 1731809035977Writing region info on filesystem at 1731809035977Initializing all the Stores at 1731809035978 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731809035978Cleaning up temporary data from old regions at 1731809035984 (+6 ms)Running coprocessor post-open hooks at 1731809035992 (+8 ms)Region opened successfully at 1731809035992 2024-11-17T02:03:55,993 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1731809034900.15c64fab21a3ca03ec0a7afabc19cdc9., pid=33, masterSystemTime=1731809035923 2024-11-17T02:03:55,994 INFO [StoreOpener-b6eba00b85aa16d5ccbfc8e3c35ff298-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b6eba00b85aa16d5ccbfc8e3c35ff298 columnFamilyName family 2024-11-17T02:03:55,994 DEBUG [StoreOpener-b6eba00b85aa16d5ccbfc8e3c35ff298-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:55,995 INFO [StoreOpener-b6eba00b85aa16d5ccbfc8e3c35ff298-1 {}] regionserver.HStore(327): Store=b6eba00b85aa16d5ccbfc8e3c35ff298/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T02:03:55,996 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for b6eba00b85aa16d5ccbfc8e3c35ff298 2024-11-17T02:03:55,997 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1731809034900.15c64fab21a3ca03ec0a7afabc19cdc9. 2024-11-17T02:03:55,997 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,ddd,1731809034900.15c64fab21a3ca03ec0a7afabc19cdc9. 2024-11-17T02:03:55,997 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,www,1731809034900.db157dde5b6770afbf869f7733d95efb. 2024-11-17T02:03:55,997 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/b6eba00b85aa16d5ccbfc8e3c35ff298 2024-11-17T02:03:55,997 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => db157dde5b6770afbf869f7733d95efb, NAME => 'test,www,1731809034900.db157dde5b6770afbf869f7733d95efb.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-17T02:03:55,997 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=15c64fab21a3ca03ec0a7afabc19cdc9, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:55,997 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test db157dde5b6770afbf869f7733d95efb 2024-11-17T02:03:55,998 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,www,1731809034900.db157dde5b6770afbf869f7733d95efb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:55,998 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for db157dde5b6770afbf869f7733d95efb 2024-11-17T02:03:55,998 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for db157dde5b6770afbf869f7733d95efb 2024-11-17T02:03:56,000 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/b6eba00b85aa16d5ccbfc8e3c35ff298 2024-11-17T02:03:56,000 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-17T02:03:56,000 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for b6eba00b85aa16d5ccbfc8e3c35ff298 2024-11-17T02:03:56,001 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for b6eba00b85aa16d5ccbfc8e3c35ff298 2024-11-17T02:03:56,001 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 15c64fab21a3ca03ec0a7afabc19cdc9, server=c6c0d7a0a7c0,45037,1731809032394 because future has completed 2024-11-17T02:03:56,005 INFO [StoreOpener-db157dde5b6770afbf869f7733d95efb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region db157dde5b6770afbf869f7733d95efb 2024-11-17T02:03:56,005 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for b6eba00b85aa16d5ccbfc8e3c35ff298 2024-11-17T02:03:56,007 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=8 2024-11-17T02:03:56,007 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 15c64fab21a3ca03ec0a7afabc19cdc9, server=c6c0d7a0a7c0,45037,1731809032394 in 220 msec 2024-11-17T02:03:56,007 INFO [StoreOpener-db157dde5b6770afbf869f7733d95efb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db157dde5b6770afbf869f7733d95efb columnFamilyName family 2024-11-17T02:03:56,008 DEBUG [StoreOpener-db157dde5b6770afbf869f7733d95efb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:56,009 INFO [StoreOpener-db157dde5b6770afbf869f7733d95efb-1 {}] regionserver.HStore(327): Store=db157dde5b6770afbf869f7733d95efb/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T02:03:56,009 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for db157dde5b6770afbf869f7733d95efb 2024-11-17T02:03:56,010 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=15c64fab21a3ca03ec0a7afabc19cdc9, ASSIGN in 415 msec 2024-11-17T02:03:56,011 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/db157dde5b6770afbf869f7733d95efb 2024-11-17T02:03:56,011 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47101, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-17T02:03:56,012 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/db157dde5b6770afbf869f7733d95efb 2024-11-17T02:03:56,013 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for db157dde5b6770afbf869f7733d95efb 2024-11-17T02:03:56,013 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for db157dde5b6770afbf869f7733d95efb 2024-11-17T02:03:56,013 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/b6eba00b85aa16d5ccbfc8e3c35ff298/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T02:03:56,014 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened b6eba00b85aa16d5ccbfc8e3c35ff298; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70881326, jitterRate=0.05621406435966492}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T02:03:56,015 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b6eba00b85aa16d5ccbfc8e3c35ff298 2024-11-17T02:03:56,015 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for b6eba00b85aa16d5ccbfc8e3c35ff298: Running coprocessor pre-open hook at 1731809035989Writing region info on filesystem at 1731809035989Initializing all the Stores at 1731809035991 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731809035991Cleaning up temporary data from old regions at 1731809036001 (+10 ms)Running coprocessor post-open hooks at 1731809036015 (+14 ms)Region opened successfully at 1731809036015 2024-11-17T02:03:56,016 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for db157dde5b6770afbf869f7733d95efb 2024-11-17T02:03:56,017 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1731809034900.b6eba00b85aa16d5ccbfc8e3c35ff298., pid=41, masterSystemTime=1731809035982 2024-11-17T02:03:56,021 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1731809034900.b6eba00b85aa16d5ccbfc8e3c35ff298. 2024-11-17T02:03:56,021 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,sss,1731809034900.b6eba00b85aa16d5ccbfc8e3c35ff298. 2024-11-17T02:03:56,021 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,lll,1731809034900.1056118c93716b24d18f749826cf316a. 2024-11-17T02:03:56,021 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => 1056118c93716b24d18f749826cf316a, NAME => 'test,lll,1731809034900.1056118c93716b24d18f749826cf316a.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-17T02:03:56,022 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=b6eba00b85aa16d5ccbfc8e3c35ff298, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,34033,1731809032455 2024-11-17T02:03:56,022 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1056118c93716b24d18f749826cf316a 2024-11-17T02:03:56,022 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,lll,1731809034900.1056118c93716b24d18f749826cf316a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:56,022 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for 1056118c93716b24d18f749826cf316a 2024-11-17T02:03:56,022 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for 1056118c93716b24d18f749826cf316a 2024-11-17T02:03:56,025 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure b6eba00b85aa16d5ccbfc8e3c35ff298, server=c6c0d7a0a7c0,34033,1731809032455 because future has completed 2024-11-17T02:03:56,027 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,uuu,1731809034900.77bd97fde6bd169f3e20553b45a65e2c. 2024-11-17T02:03:56,027 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 77bd97fde6bd169f3e20553b45a65e2c, NAME => 'test,uuu,1731809034900.77bd97fde6bd169f3e20553b45a65e2c.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-17T02:03:56,027 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 77bd97fde6bd169f3e20553b45a65e2c 2024-11-17T02:03:56,027 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,uuu,1731809034900.77bd97fde6bd169f3e20553b45a65e2c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:56,028 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 77bd97fde6bd169f3e20553b45a65e2c 2024-11-17T02:03:56,028 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 77bd97fde6bd169f3e20553b45a65e2c 2024-11-17T02:03:56,029 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/db157dde5b6770afbf869f7733d95efb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T02:03:56,031 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened db157dde5b6770afbf869f7733d95efb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60515621, jitterRate=-0.0982469767332077}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T02:03:56,031 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for db157dde5b6770afbf869f7733d95efb 2024-11-17T02:03:56,031 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for db157dde5b6770afbf869f7733d95efb: Running coprocessor pre-open hook at 1731809035998Writing region info on filesystem at 1731809035998Initializing all the Stores at 1731809036000 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731809036001 (+1 ms)Cleaning up temporary data from old regions at 1731809036013 (+12 ms)Running coprocessor post-open hooks at 1731809036031 (+18 ms)Region opened successfully at 1731809036031 2024-11-17T02:03:56,031 INFO [StoreOpener-1056118c93716b24d18f749826cf316a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1056118c93716b24d18f749826cf316a 2024-11-17T02:03:56,032 INFO [StoreOpener-77bd97fde6bd169f3e20553b45a65e2c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 77bd97fde6bd169f3e20553b45a65e2c 2024-11-17T02:03:56,032 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1731809034900.db157dde5b6770afbf869f7733d95efb., pid=38, masterSystemTime=1731809035923 2024-11-17T02:03:56,035 INFO [StoreOpener-1056118c93716b24d18f749826cf316a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1056118c93716b24d18f749826cf316a columnFamilyName family 2024-11-17T02:03:56,036 DEBUG [StoreOpener-1056118c93716b24d18f749826cf316a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:56,036 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1731809034900.db157dde5b6770afbf869f7733d95efb. 2024-11-17T02:03:56,037 INFO [StoreOpener-77bd97fde6bd169f3e20553b45a65e2c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 77bd97fde6bd169f3e20553b45a65e2c columnFamilyName family 2024-11-17T02:03:56,037 INFO [StoreOpener-1056118c93716b24d18f749826cf316a-1 {}] regionserver.HStore(327): Store=1056118c93716b24d18f749826cf316a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T02:03:56,037 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,www,1731809034900.db157dde5b6770afbf869f7733d95efb. 2024-11-17T02:03:56,037 DEBUG [StoreOpener-77bd97fde6bd169f3e20553b45a65e2c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:56,037 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for 1056118c93716b24d18f749826cf316a 2024-11-17T02:03:56,037 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,kkk,1731809034900.ac10d67ff6c3f527cc0861e738032bdb. 2024-11-17T02:03:56,038 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=23 2024-11-17T02:03:56,038 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure b6eba00b85aa16d5ccbfc8e3c35ff298, server=c6c0d7a0a7c0,34033,1731809032455 in 195 msec 2024-11-17T02:03:56,038 INFO [StoreOpener-77bd97fde6bd169f3e20553b45a65e2c-1 {}] regionserver.HStore(327): Store=77bd97fde6bd169f3e20553b45a65e2c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T02:03:56,037 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => ac10d67ff6c3f527cc0861e738032bdb, NAME => 'test,kkk,1731809034900.ac10d67ff6c3f527cc0861e738032bdb.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-17T02:03:56,038 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 77bd97fde6bd169f3e20553b45a65e2c 2024-11-17T02:03:56,038 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/1056118c93716b24d18f749826cf316a 2024-11-17T02:03:56,038 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ac10d67ff6c3f527cc0861e738032bdb 2024-11-17T02:03:56,038 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,kkk,1731809034900.ac10d67ff6c3f527cc0861e738032bdb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:56,038 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for ac10d67ff6c3f527cc0861e738032bdb 2024-11-17T02:03:56,038 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for ac10d67ff6c3f527cc0861e738032bdb 2024-11-17T02:03:56,039 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/1056118c93716b24d18f749826cf316a 2024-11-17T02:03:56,039 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/77bd97fde6bd169f3e20553b45a65e2c 2024-11-17T02:03:56,040 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for 1056118c93716b24d18f749826cf316a 2024-11-17T02:03:56,040 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for 1056118c93716b24d18f749826cf316a 2024-11-17T02:03:56,040 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/77bd97fde6bd169f3e20553b45a65e2c 2024-11-17T02:03:56,041 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 77bd97fde6bd169f3e20553b45a65e2c 2024-11-17T02:03:56,041 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 77bd97fde6bd169f3e20553b45a65e2c 2024-11-17T02:03:56,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN in 447 msec 2024-11-17T02:03:56,044 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for 1056118c93716b24d18f749826cf316a 2024-11-17T02:03:56,044 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=db157dde5b6770afbf869f7733d95efb, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:56,044 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 77bd97fde6bd169f3e20553b45a65e2c 2024-11-17T02:03:56,044 INFO [StoreOpener-ac10d67ff6c3f527cc0861e738032bdb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ac10d67ff6c3f527cc0861e738032bdb 2024-11-17T02:03:56,049 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure db157dde5b6770afbf869f7733d95efb, server=c6c0d7a0a7c0,45037,1731809032394 because future has completed 2024-11-17T02:03:56,049 INFO [StoreOpener-ac10d67ff6c3f527cc0861e738032bdb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ac10d67ff6c3f527cc0861e738032bdb columnFamilyName family 2024-11-17T02:03:56,049 DEBUG [StoreOpener-ac10d67ff6c3f527cc0861e738032bdb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:56,050 INFO [StoreOpener-ac10d67ff6c3f527cc0861e738032bdb-1 {}] regionserver.HStore(327): Store=ac10d67ff6c3f527cc0861e738032bdb/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T02:03:56,050 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for ac10d67ff6c3f527cc0861e738032bdb 2024-11-17T02:03:56,052 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/ac10d67ff6c3f527cc0861e738032bdb 2024-11-17T02:03:56,052 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/ac10d67ff6c3f527cc0861e738032bdb 2024-11-17T02:03:56,053 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for ac10d67ff6c3f527cc0861e738032bdb 2024-11-17T02:03:56,053 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for ac10d67ff6c3f527cc0861e738032bdb 2024-11-17T02:03:56,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=27 2024-11-17T02:03:56,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure db157dde5b6770afbf869f7733d95efb, server=c6c0d7a0a7c0,45037,1731809032394 in 240 msec 2024-11-17T02:03:56,056 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for ac10d67ff6c3f527cc0861e738032bdb 2024-11-17T02:03:56,058 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=db157dde5b6770afbf869f7733d95efb, ASSIGN in 463 msec 2024-11-17T02:03:56,059 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/1056118c93716b24d18f749826cf316a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T02:03:56,059 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/ac10d67ff6c3f527cc0861e738032bdb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T02:03:56,060 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/77bd97fde6bd169f3e20553b45a65e2c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T02:03:56,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T02:03:56,061 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened ac10d67ff6c3f527cc0861e738032bdb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68152185, jitterRate=0.01554669439792633}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T02:03:56,061 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened 1056118c93716b24d18f749826cf316a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61555479, jitterRate=-0.08275188505649567}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T02:03:56,061 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ac10d67ff6c3f527cc0861e738032bdb 2024-11-17T02:03:56,061 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1056118c93716b24d18f749826cf316a 2024-11-17T02:03:56,061 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 77bd97fde6bd169f3e20553b45a65e2c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66296421, jitterRate=-0.012106344103813171}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T02:03:56,061 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for ac10d67ff6c3f527cc0861e738032bdb: Running coprocessor pre-open hook at 1731809036039Writing region info on filesystem at 1731809036039Initializing all the Stores at 1731809036040 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731809036040Cleaning up temporary data from old regions at 1731809036053 (+13 ms)Running coprocessor post-open hooks at 1731809036061 (+8 ms)Region opened successfully at 1731809036061 2024-11-17T02:03:56,061 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for 1056118c93716b24d18f749826cf316a: Running coprocessor pre-open hook at 1731809036022Writing region info on filesystem at 1731809036022Initializing all the Stores at 1731809036025 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731809036025Cleaning up temporary data from old regions at 1731809036040 (+15 ms)Running coprocessor post-open hooks at 1731809036061 (+21 ms)Region opened successfully at 1731809036061 2024-11-17T02:03:56,061 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 77bd97fde6bd169f3e20553b45a65e2c 2024-11-17T02:03:56,061 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 77bd97fde6bd169f3e20553b45a65e2c: Running coprocessor pre-open hook at 1731809036028Writing region info on filesystem at 1731809036028Initializing all the Stores at 1731809036030 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731809036030Cleaning up temporary data from old regions at 1731809036041 (+11 ms)Running coprocessor post-open hooks at 1731809036061 (+20 ms)Region opened successfully at 1731809036061 2024-11-17T02:03:56,063 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1731809034900.ac10d67ff6c3f527cc0861e738032bdb., pid=35, masterSystemTime=1731809035923 2024-11-17T02:03:56,063 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1731809034900.1056118c93716b24d18f749826cf316a., pid=49, masterSystemTime=1731809035982 2024-11-17T02:03:56,063 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1731809034900.77bd97fde6bd169f3e20553b45a65e2c., pid=43, masterSystemTime=1731809036000 2024-11-17T02:03:56,066 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1731809034900.ac10d67ff6c3f527cc0861e738032bdb. 2024-11-17T02:03:56,066 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,kkk,1731809034900.ac10d67ff6c3f527cc0861e738032bdb. 2024-11-17T02:03:56,067 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,zzz,1731809034900.037b7a93730957425bc54697b3594bb2. 2024-11-17T02:03:56,067 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 037b7a93730957425bc54697b3594bb2, NAME => 'test,zzz,1731809034900.037b7a93730957425bc54697b3594bb2.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-17T02:03:56,068 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=ac10d67ff6c3f527cc0861e738032bdb, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:56,068 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 037b7a93730957425bc54697b3594bb2 2024-11-17T02:03:56,068 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,zzz,1731809034900.037b7a93730957425bc54697b3594bb2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:56,068 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 037b7a93730957425bc54697b3594bb2 2024-11-17T02:03:56,068 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 037b7a93730957425bc54697b3594bb2 2024-11-17T02:03:56,069 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1731809034900.77bd97fde6bd169f3e20553b45a65e2c. 2024-11-17T02:03:56,069 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,uuu,1731809034900.77bd97fde6bd169f3e20553b45a65e2c. 2024-11-17T02:03:56,069 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,mmm,1731809034900.663bac36df6c9873ad776bb3ad42409f. 2024-11-17T02:03:56,070 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => 663bac36df6c9873ad776bb3ad42409f, NAME => 'test,mmm,1731809034900.663bac36df6c9873ad776bb3ad42409f.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-17T02:03:56,071 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 663bac36df6c9873ad776bb3ad42409f 2024-11-17T02:03:56,071 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,mmm,1731809034900.663bac36df6c9873ad776bb3ad42409f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:56,071 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for 663bac36df6c9873ad776bb3ad42409f 2024-11-17T02:03:56,071 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for 663bac36df6c9873ad776bb3ad42409f 2024-11-17T02:03:56,071 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure ac10d67ff6c3f527cc0861e738032bdb, server=c6c0d7a0a7c0,45037,1731809032394 because future has completed 2024-11-17T02:03:56,072 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1731809034900.1056118c93716b24d18f749826cf316a. 2024-11-17T02:03:56,072 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,lll,1731809034900.1056118c93716b24d18f749826cf316a. 2024-11-17T02:03:56,072 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,,1731809034900.de9924be32211cddceef43e5b4866dc8. 2024-11-17T02:03:56,072 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => de9924be32211cddceef43e5b4866dc8, NAME => 'test,,1731809034900.de9924be32211cddceef43e5b4866dc8.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-17T02:03:56,073 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1056118c93716b24d18f749826cf316a, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,34033,1731809032455 2024-11-17T02:03:56,073 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test de9924be32211cddceef43e5b4866dc8 2024-11-17T02:03:56,073 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,,1731809034900.de9924be32211cddceef43e5b4866dc8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:56,073 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for de9924be32211cddceef43e5b4866dc8 2024-11-17T02:03:56,073 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for de9924be32211cddceef43e5b4866dc8 2024-11-17T02:03:56,073 INFO [StoreOpener-663bac36df6c9873ad776bb3ad42409f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 663bac36df6c9873ad776bb3ad42409f 2024-11-17T02:03:56,074 INFO [StoreOpener-037b7a93730957425bc54697b3594bb2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 037b7a93730957425bc54697b3594bb2 2024-11-17T02:03:56,076 INFO [StoreOpener-663bac36df6c9873ad776bb3ad42409f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 663bac36df6c9873ad776bb3ad42409f columnFamilyName family 2024-11-17T02:03:56,076 DEBUG [StoreOpener-663bac36df6c9873ad776bb3ad42409f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:56,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1056118c93716b24d18f749826cf316a, server=c6c0d7a0a7c0,34033,1731809032455 because future has completed 2024-11-17T02:03:56,077 INFO [StoreOpener-663bac36df6c9873ad776bb3ad42409f-1 {}] regionserver.HStore(327): Store=663bac36df6c9873ad776bb3ad42409f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T02:03:56,077 INFO [StoreOpener-de9924be32211cddceef43e5b4866dc8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region de9924be32211cddceef43e5b4866dc8 2024-11-17T02:03:56,077 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for 663bac36df6c9873ad776bb3ad42409f 2024-11-17T02:03:56,078 INFO [StoreOpener-037b7a93730957425bc54697b3594bb2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 037b7a93730957425bc54697b3594bb2 columnFamilyName family 2024-11-17T02:03:56,078 DEBUG [StoreOpener-037b7a93730957425bc54697b3594bb2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:56,079 INFO [StoreOpener-037b7a93730957425bc54697b3594bb2-1 {}] regionserver.HStore(327): Store=037b7a93730957425bc54697b3594bb2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T02:03:56,079 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/663bac36df6c9873ad776bb3ad42409f 2024-11-17T02:03:56,079 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 037b7a93730957425bc54697b3594bb2 2024-11-17T02:03:56,080 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/663bac36df6c9873ad776bb3ad42409f 2024-11-17T02:03:56,080 INFO [StoreOpener-de9924be32211cddceef43e5b4866dc8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de9924be32211cddceef43e5b4866dc8 columnFamilyName family 2024-11-17T02:03:56,080 DEBUG [StoreOpener-de9924be32211cddceef43e5b4866dc8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:56,080 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/037b7a93730957425bc54697b3594bb2 2024-11-17T02:03:56,080 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for 663bac36df6c9873ad776bb3ad42409f 2024-11-17T02:03:56,080 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for 663bac36df6c9873ad776bb3ad42409f 2024-11-17T02:03:56,081 INFO [StoreOpener-de9924be32211cddceef43e5b4866dc8-1 {}] regionserver.HStore(327): Store=de9924be32211cddceef43e5b4866dc8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T02:03:56,081 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/037b7a93730957425bc54697b3594bb2 2024-11-17T02:03:56,082 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 037b7a93730957425bc54697b3594bb2 2024-11-17T02:03:56,082 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for de9924be32211cddceef43e5b4866dc8 2024-11-17T02:03:56,082 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 037b7a93730957425bc54697b3594bb2 2024-11-17T02:03:56,083 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/de9924be32211cddceef43e5b4866dc8 2024-11-17T02:03:56,084 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=15 2024-11-17T02:03:56,084 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for 663bac36df6c9873ad776bb3ad42409f 2024-11-17T02:03:56,084 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure ac10d67ff6c3f527cc0861e738032bdb, server=c6c0d7a0a7c0,45037,1731809032394 in 284 msec 2024-11-17T02:03:56,085 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/de9924be32211cddceef43e5b4866dc8 2024-11-17T02:03:56,086 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for de9924be32211cddceef43e5b4866dc8 2024-11-17T02:03:56,087 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for de9924be32211cddceef43e5b4866dc8 2024-11-17T02:03:56,087 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 037b7a93730957425bc54697b3594bb2 2024-11-17T02:03:56,088 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ac10d67ff6c3f527cc0861e738032bdb, ASSIGN in 492 msec 2024-11-17T02:03:56,090 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/663bac36df6c9873ad776bb3ad42409f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T02:03:56,091 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened 663bac36df6c9873ad776bb3ad42409f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69136514, jitterRate=0.0302143394947052}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T02:03:56,091 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 663bac36df6c9873ad776bb3ad42409f 2024-11-17T02:03:56,091 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for 663bac36df6c9873ad776bb3ad42409f: Running coprocessor pre-open hook at 1731809036071Writing region info on filesystem at 1731809036071Initializing all the Stores at 1731809036073 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731809036073Cleaning up temporary data from old regions at 1731809036080 (+7 ms)Running coprocessor post-open hooks at 1731809036091 (+11 ms)Region opened successfully at 1731809036091 2024-11-17T02:03:56,092 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/037b7a93730957425bc54697b3594bb2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T02:03:56,093 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for de9924be32211cddceef43e5b4866dc8 2024-11-17T02:03:56,094 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1731809034900.663bac36df6c9873ad776bb3ad42409f., pid=52, masterSystemTime=1731809036000 2024-11-17T02:03:56,093 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 037b7a93730957425bc54697b3594bb2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73930647, jitterRate=0.10165248811244965}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T02:03:56,094 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 037b7a93730957425bc54697b3594bb2 2024-11-17T02:03:56,094 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 037b7a93730957425bc54697b3594bb2: Running coprocessor pre-open hook at 1731809036068Writing region info on filesystem at 1731809036068Initializing all the Stores at 1731809036072 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731809036074 (+2 ms)Cleaning up temporary data from old regions at 1731809036082 (+8 ms)Running coprocessor post-open hooks at 1731809036094 (+12 ms)Region opened successfully at 1731809036094 2024-11-17T02:03:56,095 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1731809034900.037b7a93730957425bc54697b3594bb2., pid=31, masterSystemTime=1731809035923 2024-11-17T02:03:56,098 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=16 2024-11-17T02:03:56,098 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1056118c93716b24d18f749826cf316a, server=c6c0d7a0a7c0,34033,1731809032455 in 211 msec 2024-11-17T02:03:56,098 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1731809034900.663bac36df6c9873ad776bb3ad42409f. 2024-11-17T02:03:56,098 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,mmm,1731809034900.663bac36df6c9873ad776bb3ad42409f. 2024-11-17T02:03:56,098 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,ccc,1731809034900.81f322eb57dcb42c3b4269a738bafd6d. 2024-11-17T02:03:56,099 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => 81f322eb57dcb42c3b4269a738bafd6d, NAME => 'test,ccc,1731809034900.81f322eb57dcb42c3b4269a738bafd6d.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-17T02:03:56,099 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/de9924be32211cddceef43e5b4866dc8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T02:03:56,099 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 81f322eb57dcb42c3b4269a738bafd6d 2024-11-17T02:03:56,099 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,ccc,1731809034900.81f322eb57dcb42c3b4269a738bafd6d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:56,099 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for 81f322eb57dcb42c3b4269a738bafd6d 2024-11-17T02:03:56,100 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for 81f322eb57dcb42c3b4269a738bafd6d 2024-11-17T02:03:56,100 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened de9924be32211cddceef43e5b4866dc8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61511543, jitterRate=-0.08340658247470856}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T02:03:56,100 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=663bac36df6c9873ad776bb3ad42409f, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:56,100 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for de9924be32211cddceef43e5b4866dc8 2024-11-17T02:03:56,100 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for de9924be32211cddceef43e5b4866dc8: Running coprocessor pre-open hook at 1731809036073Writing region info on filesystem at 1731809036073Initializing all the Stores at 1731809036075 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731809036075Cleaning up temporary data from old regions at 1731809036087 (+12 ms)Running coprocessor post-open hooks at 1731809036100 (+13 ms)Region opened successfully at 1731809036100 2024-11-17T02:03:56,102 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1731809034900.de9924be32211cddceef43e5b4866dc8., pid=47, masterSystemTime=1731809035982 2024-11-17T02:03:56,103 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1731809034900.037b7a93730957425bc54697b3594bb2. 2024-11-17T02:03:56,103 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,zzz,1731809034900.037b7a93730957425bc54697b3594bb2. 2024-11-17T02:03:56,103 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,nnn,1731809034900.74108f0b014075d4993f0eee396e39e5. 2024-11-17T02:03:56,103 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 74108f0b014075d4993f0eee396e39e5, NAME => 'test,nnn,1731809034900.74108f0b014075d4993f0eee396e39e5.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-17T02:03:56,104 INFO [StoreOpener-81f322eb57dcb42c3b4269a738bafd6d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 81f322eb57dcb42c3b4269a738bafd6d 2024-11-17T02:03:56,104 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 74108f0b014075d4993f0eee396e39e5 2024-11-17T02:03:56,104 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,nnn,1731809034900.74108f0b014075d4993f0eee396e39e5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:56,104 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 74108f0b014075d4993f0eee396e39e5 2024-11-17T02:03:56,104 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 663bac36df6c9873ad776bb3ad42409f, server=c6c0d7a0a7c0,33127,1731809032263 because future has completed 2024-11-17T02:03:56,105 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 74108f0b014075d4993f0eee396e39e5 2024-11-17T02:03:56,106 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1731809034900.de9924be32211cddceef43e5b4866dc8. 2024-11-17T02:03:56,106 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,,1731809034900.de9924be32211cddceef43e5b4866dc8. 2024-11-17T02:03:56,106 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,ooo,1731809034900.25a67f85380b4a3cd6890ab81d0b3a27. 2024-11-17T02:03:56,106 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => 25a67f85380b4a3cd6890ab81d0b3a27, NAME => 'test,ooo,1731809034900.25a67f85380b4a3cd6890ab81d0b3a27.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-17T02:03:56,107 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 25a67f85380b4a3cd6890ab81d0b3a27 2024-11-17T02:03:56,107 INFO [StoreOpener-81f322eb57dcb42c3b4269a738bafd6d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81f322eb57dcb42c3b4269a738bafd6d columnFamilyName family 2024-11-17T02:03:56,107 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,ooo,1731809034900.25a67f85380b4a3cd6890ab81d0b3a27.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:56,107 DEBUG [StoreOpener-81f322eb57dcb42c3b4269a738bafd6d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:56,107 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for 25a67f85380b4a3cd6890ab81d0b3a27 2024-11-17T02:03:56,107 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for 25a67f85380b4a3cd6890ab81d0b3a27 2024-11-17T02:03:56,107 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=de9924be32211cddceef43e5b4866dc8, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,34033,1731809032455 2024-11-17T02:03:56,108 INFO [StoreOpener-74108f0b014075d4993f0eee396e39e5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 74108f0b014075d4993f0eee396e39e5 2024-11-17T02:03:56,108 INFO [StoreOpener-81f322eb57dcb42c3b4269a738bafd6d-1 {}] regionserver.HStore(327): Store=81f322eb57dcb42c3b4269a738bafd6d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T02:03:56,109 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for 81f322eb57dcb42c3b4269a738bafd6d 2024-11-17T02:03:56,109 INFO [StoreOpener-25a67f85380b4a3cd6890ab81d0b3a27-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 25a67f85380b4a3cd6890ab81d0b3a27 2024-11-17T02:03:56,110 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/81f322eb57dcb42c3b4269a738bafd6d 2024-11-17T02:03:56,111 INFO [StoreOpener-74108f0b014075d4993f0eee396e39e5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 74108f0b014075d4993f0eee396e39e5 columnFamilyName family 2024-11-17T02:03:56,111 DEBUG [StoreOpener-74108f0b014075d4993f0eee396e39e5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:56,111 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/81f322eb57dcb42c3b4269a738bafd6d 2024-11-17T02:03:56,111 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure de9924be32211cddceef43e5b4866dc8, server=c6c0d7a0a7c0,34033,1731809032455 because future has completed 2024-11-17T02:03:56,112 INFO [StoreOpener-74108f0b014075d4993f0eee396e39e5-1 {}] regionserver.HStore(327): Store=74108f0b014075d4993f0eee396e39e5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T02:03:56,112 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for 81f322eb57dcb42c3b4269a738bafd6d 2024-11-17T02:03:56,112 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for 81f322eb57dcb42c3b4269a738bafd6d 2024-11-17T02:03:56,112 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 74108f0b014075d4993f0eee396e39e5 2024-11-17T02:03:56,113 INFO [StoreOpener-25a67f85380b4a3cd6890ab81d0b3a27-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25a67f85380b4a3cd6890ab81d0b3a27 columnFamilyName family 2024-11-17T02:03:56,113 DEBUG [StoreOpener-25a67f85380b4a3cd6890ab81d0b3a27-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:56,113 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/74108f0b014075d4993f0eee396e39e5 2024-11-17T02:03:56,114 INFO [StoreOpener-25a67f85380b4a3cd6890ab81d0b3a27-1 {}] regionserver.HStore(327): Store=25a67f85380b4a3cd6890ab81d0b3a27/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T02:03:56,114 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/74108f0b014075d4993f0eee396e39e5 2024-11-17T02:03:56,114 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for 25a67f85380b4a3cd6890ab81d0b3a27 2024-11-17T02:03:56,115 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 74108f0b014075d4993f0eee396e39e5 2024-11-17T02:03:56,115 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 74108f0b014075d4993f0eee396e39e5 2024-11-17T02:03:56,115 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/25a67f85380b4a3cd6890ab81d0b3a27 2024-11-17T02:03:56,116 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/25a67f85380b4a3cd6890ab81d0b3a27 2024-11-17T02:03:56,117 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for 25a67f85380b4a3cd6890ab81d0b3a27 2024-11-17T02:03:56,117 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for 81f322eb57dcb42c3b4269a738bafd6d 2024-11-17T02:03:56,117 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for 25a67f85380b4a3cd6890ab81d0b3a27 2024-11-17T02:03:56,119 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 74108f0b014075d4993f0eee396e39e5 2024-11-17T02:03:56,120 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for 25a67f85380b4a3cd6890ab81d0b3a27 2024-11-17T02:03:56,121 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=5 2024-11-17T02:03:56,121 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure de9924be32211cddceef43e5b4866dc8, server=c6c0d7a0a7c0,34033,1731809032455 in 245 msec 2024-11-17T02:03:56,124 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/81f322eb57dcb42c3b4269a738bafd6d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T02:03:56,125 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/74108f0b014075d4993f0eee396e39e5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T02:03:56,125 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN in 529 msec 2024-11-17T02:03:56,125 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened 81f322eb57dcb42c3b4269a738bafd6d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61958200, jitterRate=-0.07675087451934814}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T02:03:56,125 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 81f322eb57dcb42c3b4269a738bafd6d 2024-11-17T02:03:56,125 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 74108f0b014075d4993f0eee396e39e5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68071284, jitterRate=0.014341175556182861}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T02:03:56,125 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 74108f0b014075d4993f0eee396e39e5 2024-11-17T02:03:56,125 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for 81f322eb57dcb42c3b4269a738bafd6d: Running coprocessor pre-open hook at 1731809036100Writing region info on filesystem at 1731809036100Initializing all the Stores at 1731809036102 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731809036102Cleaning up temporary data from old regions at 1731809036112 (+10 ms)Running coprocessor post-open hooks at 1731809036125 (+13 ms)Region opened successfully at 1731809036125 2024-11-17T02:03:56,126 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 74108f0b014075d4993f0eee396e39e5: Running coprocessor pre-open hook at 1731809036105Writing region info on filesystem at 1731809036105Initializing all the Stores at 1731809036107 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731809036107Cleaning up temporary data from old regions at 1731809036115 (+8 ms)Running coprocessor post-open hooks at 1731809036125 (+10 ms)Region opened successfully at 1731809036126 (+1 ms) 2024-11-17T02:03:56,127 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1731809034900.74108f0b014075d4993f0eee396e39e5., pid=32, masterSystemTime=1731809035923 2024-11-17T02:03:56,127 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1731809034900.81f322eb57dcb42c3b4269a738bafd6d., pid=54, masterSystemTime=1731809036000 2024-11-17T02:03:56,129 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1731809034900.81f322eb57dcb42c3b4269a738bafd6d. 2024-11-17T02:03:56,130 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,ccc,1731809034900.81f322eb57dcb42c3b4269a738bafd6d. 2024-11-17T02:03:56,130 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,ggg,1731809034900.7517bce9e04879764302cccd45af20fb. 2024-11-17T02:03:56,130 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 7517bce9e04879764302cccd45af20fb, NAME => 'test,ggg,1731809034900.7517bce9e04879764302cccd45af20fb.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-17T02:03:56,131 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1731809034900.74108f0b014075d4993f0eee396e39e5. 2024-11-17T02:03:56,131 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,nnn,1731809034900.74108f0b014075d4993f0eee396e39e5. 2024-11-17T02:03:56,131 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,ttt,1731809034900.4748cc4c5ad3838a3d0568cee797c169. 2024-11-17T02:03:56,131 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 4748cc4c5ad3838a3d0568cee797c169, NAME => 'test,ttt,1731809034900.4748cc4c5ad3838a3d0568cee797c169.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-17T02:03:56,132 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/25a67f85380b4a3cd6890ab81d0b3a27/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T02:03:56,133 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened 25a67f85380b4a3cd6890ab81d0b3a27; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65884353, jitterRate=-0.018246635794639587}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T02:03:56,133 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 25a67f85380b4a3cd6890ab81d0b3a27 2024-11-17T02:03:56,133 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for 25a67f85380b4a3cd6890ab81d0b3a27: Running coprocessor pre-open hook at 1731809036107Writing region info on filesystem at 1731809036107Initializing all the Stores at 1731809036108 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731809036108Cleaning up temporary data from old regions at 1731809036117 (+9 ms)Running coprocessor post-open hooks at 1731809036133 (+16 ms)Region opened successfully at 1731809036133 2024-11-17T02:03:56,134 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=17 2024-11-17T02:03:56,134 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 663bac36df6c9873ad776bb3ad42409f, server=c6c0d7a0a7c0,33127,1731809032263 in 228 msec 2024-11-17T02:03:56,134 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4748cc4c5ad3838a3d0568cee797c169 2024-11-17T02:03:56,134 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7517bce9e04879764302cccd45af20fb 2024-11-17T02:03:56,135 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,ggg,1731809034900.7517bce9e04879764302cccd45af20fb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:56,135 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,ttt,1731809034900.4748cc4c5ad3838a3d0568cee797c169.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:56,135 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 4748cc4c5ad3838a3d0568cee797c169 2024-11-17T02:03:56,135 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 7517bce9e04879764302cccd45af20fb 2024-11-17T02:03:56,135 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 7517bce9e04879764302cccd45af20fb 2024-11-17T02:03:56,135 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 4748cc4c5ad3838a3d0568cee797c169 2024-11-17T02:03:56,139 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1731809034900.25a67f85380b4a3cd6890ab81d0b3a27., pid=50, masterSystemTime=1731809035982 2024-11-17T02:03:56,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=663bac36df6c9873ad776bb3ad42409f, ASSIGN in 542 msec 2024-11-17T02:03:56,146 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=74108f0b014075d4993f0eee396e39e5, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:56,146 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1731809034900.25a67f85380b4a3cd6890ab81d0b3a27. 2024-11-17T02:03:56,147 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,ooo,1731809034900.25a67f85380b4a3cd6890ab81d0b3a27. 2024-11-17T02:03:56,147 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,yyy,1731809034900.7eed07a820ab15d7af64462e2c1b70f7. 2024-11-17T02:03:56,147 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 7eed07a820ab15d7af64462e2c1b70f7, NAME => 'test,yyy,1731809034900.7eed07a820ab15d7af64462e2c1b70f7.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-17T02:03:56,147 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7eed07a820ab15d7af64462e2c1b70f7 2024-11-17T02:03:56,147 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,yyy,1731809034900.7eed07a820ab15d7af64462e2c1b70f7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:56,148 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 7eed07a820ab15d7af64462e2c1b70f7 2024-11-17T02:03:56,148 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 7eed07a820ab15d7af64462e2c1b70f7 2024-11-17T02:03:56,148 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32803 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=c6c0d7a0a7c0,45037,1731809032394, table=test, region=74108f0b014075d4993f0eee396e39e5. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-17T02:03:56,148 INFO [StoreOpener-4748cc4c5ad3838a3d0568cee797c169-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4748cc4c5ad3838a3d0568cee797c169 2024-11-17T02:03:56,148 INFO [StoreOpener-7517bce9e04879764302cccd45af20fb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7517bce9e04879764302cccd45af20fb 2024-11-17T02:03:56,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 74108f0b014075d4993f0eee396e39e5, server=c6c0d7a0a7c0,45037,1731809032394 because future has completed 2024-11-17T02:03:56,151 INFO [StoreOpener-4748cc4c5ad3838a3d0568cee797c169-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4748cc4c5ad3838a3d0568cee797c169 columnFamilyName family 2024-11-17T02:03:56,151 INFO [StoreOpener-7517bce9e04879764302cccd45af20fb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7517bce9e04879764302cccd45af20fb columnFamilyName family 2024-11-17T02:03:56,151 DEBUG [StoreOpener-4748cc4c5ad3838a3d0568cee797c169-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:56,151 DEBUG [StoreOpener-7517bce9e04879764302cccd45af20fb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:56,152 INFO [StoreOpener-4748cc4c5ad3838a3d0568cee797c169-1 {}] regionserver.HStore(327): Store=4748cc4c5ad3838a3d0568cee797c169/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T02:03:56,152 INFO [StoreOpener-7517bce9e04879764302cccd45af20fb-1 {}] regionserver.HStore(327): Store=7517bce9e04879764302cccd45af20fb/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T02:03:56,152 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 4748cc4c5ad3838a3d0568cee797c169 2024-11-17T02:03:56,152 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 7517bce9e04879764302cccd45af20fb 2024-11-17T02:03:56,153 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=25a67f85380b4a3cd6890ab81d0b3a27, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,34033,1731809032455 2024-11-17T02:03:56,153 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/7517bce9e04879764302cccd45af20fb 2024-11-17T02:03:56,153 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/4748cc4c5ad3838a3d0568cee797c169 2024-11-17T02:03:56,154 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/7517bce9e04879764302cccd45af20fb 2024-11-17T02:03:56,154 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/4748cc4c5ad3838a3d0568cee797c169 2024-11-17T02:03:56,155 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 7517bce9e04879764302cccd45af20fb 2024-11-17T02:03:56,155 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 7517bce9e04879764302cccd45af20fb 2024-11-17T02:03:56,155 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 4748cc4c5ad3838a3d0568cee797c169 2024-11-17T02:03:56,155 INFO [StoreOpener-7eed07a820ab15d7af64462e2c1b70f7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7eed07a820ab15d7af64462e2c1b70f7 2024-11-17T02:03:56,156 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 4748cc4c5ad3838a3d0568cee797c169 2024-11-17T02:03:56,157 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 25a67f85380b4a3cd6890ab81d0b3a27, server=c6c0d7a0a7c0,34033,1731809032455 because future has completed 2024-11-17T02:03:56,158 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 7517bce9e04879764302cccd45af20fb 2024-11-17T02:03:56,158 INFO [StoreOpener-7eed07a820ab15d7af64462e2c1b70f7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7eed07a820ab15d7af64462e2c1b70f7 columnFamilyName family 2024-11-17T02:03:56,159 DEBUG [StoreOpener-7eed07a820ab15d7af64462e2c1b70f7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:56,159 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 4748cc4c5ad3838a3d0568cee797c169 2024-11-17T02:03:56,159 INFO [StoreOpener-7eed07a820ab15d7af64462e2c1b70f7-1 {}] regionserver.HStore(327): Store=7eed07a820ab15d7af64462e2c1b70f7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T02:03:56,160 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 7eed07a820ab15d7af64462e2c1b70f7 2024-11-17T02:03:56,161 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/7eed07a820ab15d7af64462e2c1b70f7 2024-11-17T02:03:56,162 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/7eed07a820ab15d7af64462e2c1b70f7 2024-11-17T02:03:56,163 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 7eed07a820ab15d7af64462e2c1b70f7 2024-11-17T02:03:56,164 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 7eed07a820ab15d7af64462e2c1b70f7 2024-11-17T02:03:56,164 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/7517bce9e04879764302cccd45af20fb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T02:03:56,165 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 7517bce9e04879764302cccd45af20fb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75399820, jitterRate=0.12354487180709839}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T02:03:56,165 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7517bce9e04879764302cccd45af20fb 2024-11-17T02:03:56,165 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 7517bce9e04879764302cccd45af20fb: Running coprocessor pre-open hook at 1731809036142Writing region info on filesystem at 1731809036142Initializing all the Stores at 1731809036144 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731809036144Cleaning up temporary data from old regions at 1731809036155 (+11 ms)Running coprocessor post-open hooks at 1731809036165 (+10 ms)Region opened successfully at 1731809036165 2024-11-17T02:03:56,166 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1731809034900.7517bce9e04879764302cccd45af20fb., pid=55, masterSystemTime=1731809036000 2024-11-17T02:03:56,167 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 7eed07a820ab15d7af64462e2c1b70f7 2024-11-17T02:03:56,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=18 2024-11-17T02:03:56,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure 74108f0b014075d4993f0eee396e39e5, server=c6c0d7a0a7c0,45037,1731809032394 in 383 msec 2024-11-17T02:03:56,169 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1731809034900.7517bce9e04879764302cccd45af20fb. 2024-11-17T02:03:56,169 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,ggg,1731809034900.7517bce9e04879764302cccd45af20fb. 2024-11-17T02:03:56,169 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,eee,1731809034900.8356c0fdb856c376b8678f2c2597565a. 2024-11-17T02:03:56,170 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => 8356c0fdb856c376b8678f2c2597565a, NAME => 'test,eee,1731809034900.8356c0fdb856c376b8678f2c2597565a.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-17T02:03:56,170 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=74108f0b014075d4993f0eee396e39e5, ASSIGN in 575 msec 2024-11-17T02:03:56,170 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8356c0fdb856c376b8678f2c2597565a 2024-11-17T02:03:56,170 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,eee,1731809034900.8356c0fdb856c376b8678f2c2597565a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:56,170 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for 8356c0fdb856c376b8678f2c2597565a 2024-11-17T02:03:56,170 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for 8356c0fdb856c376b8678f2c2597565a 2024-11-17T02:03:56,172 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=7517bce9e04879764302cccd45af20fb, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:56,174 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7517bce9e04879764302cccd45af20fb, server=c6c0d7a0a7c0,33127,1731809032263 because future has completed 2024-11-17T02:03:56,179 INFO [StoreOpener-8356c0fdb856c376b8678f2c2597565a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8356c0fdb856c376b8678f2c2597565a 2024-11-17T02:03:56,180 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/4748cc4c5ad3838a3d0568cee797c169/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T02:03:56,180 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=19 2024-11-17T02:03:56,180 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/7eed07a820ab15d7af64462e2c1b70f7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T02:03:56,181 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 25a67f85380b4a3cd6890ab81d0b3a27, server=c6c0d7a0a7c0,34033,1731809032455 in 291 msec 2024-11-17T02:03:56,181 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 4748cc4c5ad3838a3d0568cee797c169; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74571747, jitterRate=0.11120562255382538}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T02:03:56,181 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4748cc4c5ad3838a3d0568cee797c169 2024-11-17T02:03:56,181 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 4748cc4c5ad3838a3d0568cee797c169: Running coprocessor pre-open hook at 1731809036142Writing region info on filesystem at 1731809036142Initializing all the Stores at 1731809036145 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731809036145Cleaning up temporary data from old regions at 1731809036156 (+11 ms)Running coprocessor post-open hooks at 1731809036181 (+25 ms)Region opened successfully at 1731809036181 2024-11-17T02:03:56,181 INFO [StoreOpener-8356c0fdb856c376b8678f2c2597565a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8356c0fdb856c376b8678f2c2597565a columnFamilyName family 2024-11-17T02:03:56,181 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 7eed07a820ab15d7af64462e2c1b70f7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61386224, jitterRate=-0.08527398109436035}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T02:03:56,181 DEBUG [StoreOpener-8356c0fdb856c376b8678f2c2597565a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:56,182 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7eed07a820ab15d7af64462e2c1b70f7 2024-11-17T02:03:56,182 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 7eed07a820ab15d7af64462e2c1b70f7: Running coprocessor pre-open hook at 1731809036148Writing region info on filesystem at 1731809036148Initializing all the Stores at 1731809036151 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731809036151Cleaning up temporary data from old regions at 1731809036164 (+13 ms)Running coprocessor post-open hooks at 1731809036182 (+18 ms)Region opened successfully at 1731809036182 2024-11-17T02:03:56,182 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1731809034900.4748cc4c5ad3838a3d0568cee797c169., pid=37, masterSystemTime=1731809035923 2024-11-17T02:03:56,183 INFO [StoreOpener-8356c0fdb856c376b8678f2c2597565a-1 {}] regionserver.HStore(327): Store=8356c0fdb856c376b8678f2c2597565a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T02:03:56,183 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for 8356c0fdb856c376b8678f2c2597565a 2024-11-17T02:03:56,184 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1731809034900.7eed07a820ab15d7af64462e2c1b70f7., pid=40, masterSystemTime=1731809035982 2024-11-17T02:03:56,184 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/8356c0fdb856c376b8678f2c2597565a 2024-11-17T02:03:56,185 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/8356c0fdb856c376b8678f2c2597565a 2024-11-17T02:03:56,185 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN in 589 msec 2024-11-17T02:03:56,185 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for 8356c0fdb856c376b8678f2c2597565a 2024-11-17T02:03:56,186 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for 8356c0fdb856c376b8678f2c2597565a 2024-11-17T02:03:56,187 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1731809034900.4748cc4c5ad3838a3d0568cee797c169. 2024-11-17T02:03:56,187 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,ttt,1731809034900.4748cc4c5ad3838a3d0568cee797c169. 2024-11-17T02:03:56,187 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,hhh,1731809034900.6fd33da09f9d938215ac0e7127ad70d5. 2024-11-17T02:03:56,187 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 6fd33da09f9d938215ac0e7127ad70d5, NAME => 'test,hhh,1731809034900.6fd33da09f9d938215ac0e7127ad70d5.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-17T02:03:56,187 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6fd33da09f9d938215ac0e7127ad70d5 2024-11-17T02:03:56,188 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,hhh,1731809034900.6fd33da09f9d938215ac0e7127ad70d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:56,188 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 6fd33da09f9d938215ac0e7127ad70d5 2024-11-17T02:03:56,188 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 6fd33da09f9d938215ac0e7127ad70d5 2024-11-17T02:03:56,189 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for 8356c0fdb856c376b8678f2c2597565a 2024-11-17T02:03:56,190 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=4748cc4c5ad3838a3d0568cee797c169, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:56,190 INFO [StoreOpener-6fd33da09f9d938215ac0e7127ad70d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6fd33da09f9d938215ac0e7127ad70d5 2024-11-17T02:03:56,191 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1731809034900.7eed07a820ab15d7af64462e2c1b70f7. 2024-11-17T02:03:56,191 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,yyy,1731809034900.7eed07a820ab15d7af64462e2c1b70f7. 2024-11-17T02:03:56,191 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,iii,1731809034900.03b7adb3afa9bebd86dc1da38125b982. 2024-11-17T02:03:56,191 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => 03b7adb3afa9bebd86dc1da38125b982, NAME => 'test,iii,1731809034900.03b7adb3afa9bebd86dc1da38125b982.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-17T02:03:56,192 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 03b7adb3afa9bebd86dc1da38125b982 2024-11-17T02:03:56,192 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,iii,1731809034900.03b7adb3afa9bebd86dc1da38125b982.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:56,192 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for 03b7adb3afa9bebd86dc1da38125b982 2024-11-17T02:03:56,192 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for 03b7adb3afa9bebd86dc1da38125b982 2024-11-17T02:03:56,193 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4748cc4c5ad3838a3d0568cee797c169, server=c6c0d7a0a7c0,45037,1731809032394 because future has completed 2024-11-17T02:03:56,194 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/8356c0fdb856c376b8678f2c2597565a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T02:03:56,194 INFO [StoreOpener-6fd33da09f9d938215ac0e7127ad70d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fd33da09f9d938215ac0e7127ad70d5 columnFamilyName family 2024-11-17T02:03:56,194 DEBUG [StoreOpener-6fd33da09f9d938215ac0e7127ad70d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:56,195 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened 8356c0fdb856c376b8678f2c2597565a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61799516, jitterRate=-0.07911545038223267}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T02:03:56,195 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8356c0fdb856c376b8678f2c2597565a 2024-11-17T02:03:56,195 INFO [StoreOpener-6fd33da09f9d938215ac0e7127ad70d5-1 {}] regionserver.HStore(327): Store=6fd33da09f9d938215ac0e7127ad70d5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T02:03:56,195 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for 8356c0fdb856c376b8678f2c2597565a: Running coprocessor pre-open hook at 1731809036171Writing region info on filesystem at 1731809036171Initializing all the Stores at 1731809036172 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731809036172Cleaning up temporary data from old regions at 1731809036186 (+14 ms)Running coprocessor post-open hooks at 1731809036195 (+9 ms)Region opened successfully at 1731809036195 2024-11-17T02:03:56,195 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=7eed07a820ab15d7af64462e2c1b70f7, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,34033,1731809032455 2024-11-17T02:03:56,195 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 6fd33da09f9d938215ac0e7127ad70d5 2024-11-17T02:03:56,197 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1731809034900.8356c0fdb856c376b8678f2c2597565a., pid=53, masterSystemTime=1731809036000 2024-11-17T02:03:56,197 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/6fd33da09f9d938215ac0e7127ad70d5 2024-11-17T02:03:56,198 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/6fd33da09f9d938215ac0e7127ad70d5 2024-11-17T02:03:56,199 INFO [StoreOpener-03b7adb3afa9bebd86dc1da38125b982-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 03b7adb3afa9bebd86dc1da38125b982 2024-11-17T02:03:56,199 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 6fd33da09f9d938215ac0e7127ad70d5 2024-11-17T02:03:56,199 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 6fd33da09f9d938215ac0e7127ad70d5 2024-11-17T02:03:56,200 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7eed07a820ab15d7af64462e2c1b70f7, server=c6c0d7a0a7c0,34033,1731809032455 because future has completed 2024-11-17T02:03:56,200 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1731809034900.8356c0fdb856c376b8678f2c2597565a. 2024-11-17T02:03:56,200 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,eee,1731809034900.8356c0fdb856c376b8678f2c2597565a. 2024-11-17T02:03:56,201 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,jjj,1731809034900.05534a4a027a1fdc90227b7a2e681933. 2024-11-17T02:03:56,201 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => 05534a4a027a1fdc90227b7a2e681933, NAME => 'test,jjj,1731809034900.05534a4a027a1fdc90227b7a2e681933.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-17T02:03:56,201 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 05534a4a027a1fdc90227b7a2e681933 2024-11-17T02:03:56,201 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,jjj,1731809034900.05534a4a027a1fdc90227b7a2e681933.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:56,201 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for 05534a4a027a1fdc90227b7a2e681933 2024-11-17T02:03:56,201 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for 05534a4a027a1fdc90227b7a2e681933 2024-11-17T02:03:56,202 INFO [StoreOpener-03b7adb3afa9bebd86dc1da38125b982-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 03b7adb3afa9bebd86dc1da38125b982 columnFamilyName family 2024-11-17T02:03:56,202 DEBUG [StoreOpener-03b7adb3afa9bebd86dc1da38125b982-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:56,202 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 6fd33da09f9d938215ac0e7127ad70d5 2024-11-17T02:03:56,203 INFO [StoreOpener-03b7adb3afa9bebd86dc1da38125b982-1 {}] regionserver.HStore(327): Store=03b7adb3afa9bebd86dc1da38125b982/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T02:03:56,203 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for 03b7adb3afa9bebd86dc1da38125b982 2024-11-17T02:03:56,204 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/03b7adb3afa9bebd86dc1da38125b982 2024-11-17T02:03:56,205 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/03b7adb3afa9bebd86dc1da38125b982 2024-11-17T02:03:56,205 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for 03b7adb3afa9bebd86dc1da38125b982 2024-11-17T02:03:56,206 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for 03b7adb3afa9bebd86dc1da38125b982 2024-11-17T02:03:56,206 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=24 2024-11-17T02:03:56,206 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 4748cc4c5ad3838a3d0568cee797c169, server=c6c0d7a0a7c0,45037,1731809032394 in 395 msec 2024-11-17T02:03:56,208 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for 03b7adb3afa9bebd86dc1da38125b982 2024-11-17T02:03:56,209 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4748cc4c5ad3838a3d0568cee797c169, ASSIGN in 614 msec 2024-11-17T02:03:56,214 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=29 2024-11-17T02:03:56,214 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 7eed07a820ab15d7af64462e2c1b70f7, server=c6c0d7a0a7c0,34033,1731809032455 in 384 msec 2024-11-17T02:03:56,215 INFO [StoreOpener-05534a4a027a1fdc90227b7a2e681933-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 05534a4a027a1fdc90227b7a2e681933 2024-11-17T02:03:56,216 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/6fd33da09f9d938215ac0e7127ad70d5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T02:03:56,216 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/03b7adb3afa9bebd86dc1da38125b982/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T02:03:56,216 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 6fd33da09f9d938215ac0e7127ad70d5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62239547, jitterRate=-0.07255847752094269}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T02:03:56,216 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6fd33da09f9d938215ac0e7127ad70d5 2024-11-17T02:03:56,217 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 6fd33da09f9d938215ac0e7127ad70d5: Running coprocessor pre-open hook at 1731809036188Writing region info on filesystem at 1731809036188Initializing all the Stores at 1731809036189 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731809036189Cleaning up temporary data from old regions at 1731809036199 (+10 ms)Running coprocessor post-open hooks at 1731809036217 (+18 ms)Region opened successfully at 1731809036217 2024-11-17T02:03:56,217 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened 03b7adb3afa9bebd86dc1da38125b982; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60604775, jitterRate=-0.09691847860813141}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T02:03:56,217 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 03b7adb3afa9bebd86dc1da38125b982 2024-11-17T02:03:56,217 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for 03b7adb3afa9bebd86dc1da38125b982: Running coprocessor pre-open hook at 1731809036192Writing region info on filesystem at 1731809036192Initializing all the Stores at 1731809036195 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731809036195Cleaning up temporary data from old regions at 1731809036206 (+11 ms)Running coprocessor post-open hooks at 1731809036217 (+11 ms)Region opened successfully at 1731809036217 2024-11-17T02:03:56,218 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1731809034900.03b7adb3afa9bebd86dc1da38125b982., pid=48, masterSystemTime=1731809035982 2024-11-17T02:03:56,218 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1731809034900.6fd33da09f9d938215ac0e7127ad70d5., pid=34, masterSystemTime=1731809035923 2024-11-17T02:03:56,218 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN in 622 msec 2024-11-17T02:03:56,218 INFO [StoreOpener-05534a4a027a1fdc90227b7a2e681933-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 05534a4a027a1fdc90227b7a2e681933 columnFamilyName family 2024-11-17T02:03:56,219 DEBUG [StoreOpener-05534a4a027a1fdc90227b7a2e681933-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:56,220 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=8356c0fdb856c376b8678f2c2597565a, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:56,221 INFO [StoreOpener-05534a4a027a1fdc90227b7a2e681933-1 {}] regionserver.HStore(327): Store=05534a4a027a1fdc90227b7a2e681933/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T02:03:56,221 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for 05534a4a027a1fdc90227b7a2e681933 2024-11-17T02:03:56,222 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1731809034900.6fd33da09f9d938215ac0e7127ad70d5. 2024-11-17T02:03:56,222 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,hhh,1731809034900.6fd33da09f9d938215ac0e7127ad70d5. 2024-11-17T02:03:56,222 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/05534a4a027a1fdc90227b7a2e681933 2024-11-17T02:03:56,223 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/05534a4a027a1fdc90227b7a2e681933 2024-11-17T02:03:56,224 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for 05534a4a027a1fdc90227b7a2e681933 2024-11-17T02:03:56,224 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for 05534a4a027a1fdc90227b7a2e681933 2024-11-17T02:03:56,224 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8356c0fdb856c376b8678f2c2597565a, server=c6c0d7a0a7c0,33127,1731809032263 because future has completed 2024-11-17T02:03:56,224 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1731809034900.03b7adb3afa9bebd86dc1da38125b982. 2024-11-17T02:03:56,224 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,iii,1731809034900.03b7adb3afa9bebd86dc1da38125b982. 2024-11-17T02:03:56,225 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,fff,1731809034900.112f5bbe15f72793585db9ba98950976. 2024-11-17T02:03:56,225 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 112f5bbe15f72793585db9ba98950976, NAME => 'test,fff,1731809034900.112f5bbe15f72793585db9ba98950976.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-17T02:03:56,225 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 112f5bbe15f72793585db9ba98950976 2024-11-17T02:03:56,225 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,fff,1731809034900.112f5bbe15f72793585db9ba98950976.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:56,226 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 112f5bbe15f72793585db9ba98950976 2024-11-17T02:03:56,226 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 112f5bbe15f72793585db9ba98950976 2024-11-17T02:03:56,226 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for 05534a4a027a1fdc90227b7a2e681933 2024-11-17T02:03:56,228 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=03b7adb3afa9bebd86dc1da38125b982, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,34033,1731809032455 2024-11-17T02:03:56,228 INFO [StoreOpener-112f5bbe15f72793585db9ba98950976-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 112f5bbe15f72793585db9ba98950976 2024-11-17T02:03:56,230 INFO [StoreOpener-112f5bbe15f72793585db9ba98950976-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 112f5bbe15f72793585db9ba98950976 columnFamilyName family 2024-11-17T02:03:56,230 DEBUG [StoreOpener-112f5bbe15f72793585db9ba98950976-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:56,231 INFO [StoreOpener-112f5bbe15f72793585db9ba98950976-1 {}] regionserver.HStore(327): Store=112f5bbe15f72793585db9ba98950976/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T02:03:56,231 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 112f5bbe15f72793585db9ba98950976 2024-11-17T02:03:56,232 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,34033,1731809032455 because future has completed 2024-11-17T02:03:56,232 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/05534a4a027a1fdc90227b7a2e681933/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T02:03:56,233 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/112f5bbe15f72793585db9ba98950976 2024-11-17T02:03:56,233 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened 05534a4a027a1fdc90227b7a2e681933; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63502810, jitterRate=-0.053734391927719116}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T02:03:56,233 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 05534a4a027a1fdc90227b7a2e681933 2024-11-17T02:03:56,233 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/112f5bbe15f72793585db9ba98950976 2024-11-17T02:03:56,233 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for 05534a4a027a1fdc90227b7a2e681933: Running coprocessor pre-open hook at 1731809036202Writing region info on filesystem at 1731809036202Initializing all the Stores at 1731809036204 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731809036204Cleaning up temporary data from old regions at 1731809036224 (+20 ms)Running coprocessor post-open hooks at 1731809036233 (+9 ms)Region opened successfully at 1731809036233 2024-11-17T02:03:56,234 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 112f5bbe15f72793585db9ba98950976 2024-11-17T02:03:56,234 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 112f5bbe15f72793585db9ba98950976 2024-11-17T02:03:56,234 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1731809034900.05534a4a027a1fdc90227b7a2e681933., pid=56, masterSystemTime=1731809036000 2024-11-17T02:03:56,238 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1731809034900.05534a4a027a1fdc90227b7a2e681933. 2024-11-17T02:03:56,238 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,jjj,1731809034900.05534a4a027a1fdc90227b7a2e681933. 2024-11-17T02:03:56,238 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,rrr,1731809034900.b7934444caa228b9e5d6d7b28b59a796. 2024-11-17T02:03:56,238 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=9 2024-11-17T02:03:56,238 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 8356c0fdb856c376b8678f2c2597565a, server=c6c0d7a0a7c0,33127,1731809032263 in 330 msec 2024-11-17T02:03:56,238 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => b7934444caa228b9e5d6d7b28b59a796, NAME => 'test,rrr,1731809034900.b7934444caa228b9e5d6d7b28b59a796.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-17T02:03:56,239 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 112f5bbe15f72793585db9ba98950976 2024-11-17T02:03:56,239 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b7934444caa228b9e5d6d7b28b59a796 2024-11-17T02:03:56,239 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,rrr,1731809034900.b7934444caa228b9e5d6d7b28b59a796.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:56,239 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for b7934444caa228b9e5d6d7b28b59a796 2024-11-17T02:03:56,239 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for b7934444caa228b9e5d6d7b28b59a796 2024-11-17T02:03:56,242 INFO [StoreOpener-b7934444caa228b9e5d6d7b28b59a796-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b7934444caa228b9e5d6d7b28b59a796 2024-11-17T02:03:56,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8356c0fdb856c376b8678f2c2597565a, ASSIGN in 646 msec 2024-11-17T02:03:56,243 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/112f5bbe15f72793585db9ba98950976/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T02:03:56,244 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=05534a4a027a1fdc90227b7a2e681933, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:56,244 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 112f5bbe15f72793585db9ba98950976; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70470687, jitterRate=0.05009506642818451}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T02:03:56,245 INFO [StoreOpener-b7934444caa228b9e5d6d7b28b59a796-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7934444caa228b9e5d6d7b28b59a796 columnFamilyName family 2024-11-17T02:03:56,245 DEBUG [StoreOpener-b7934444caa228b9e5d6d7b28b59a796-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:56,246 INFO [StoreOpener-b7934444caa228b9e5d6d7b28b59a796-1 {}] regionserver.HStore(327): Store=b7934444caa228b9e5d6d7b28b59a796/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T02:03:56,246 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for b7934444caa228b9e5d6d7b28b59a796 2024-11-17T02:03:56,248 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/b7934444caa228b9e5d6d7b28b59a796 2024-11-17T02:03:56,248 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 112f5bbe15f72793585db9ba98950976 2024-11-17T02:03:56,248 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 112f5bbe15f72793585db9ba98950976: Running coprocessor pre-open hook at 1731809036226Writing region info on filesystem at 1731809036226Initializing all the Stores at 1731809036227 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731809036227Cleaning up temporary data from old regions at 1731809036234 (+7 ms)Running coprocessor post-open hooks at 1731809036248 (+14 ms)Region opened successfully at 1731809036248 2024-11-17T02:03:56,249 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 05534a4a027a1fdc90227b7a2e681933, server=c6c0d7a0a7c0,33127,1731809032263 because future has completed 2024-11-17T02:03:56,249 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32803 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=c6c0d7a0a7c0,33127,1731809032263, table=test, region=7517bce9e04879764302cccd45af20fb. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-17T02:03:56,249 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32803 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=c6c0d7a0a7c0,33127,1731809032263, table=test, region=05534a4a027a1fdc90227b7a2e681933. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-17T02:03:56,250 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/b7934444caa228b9e5d6d7b28b59a796 2024-11-17T02:03:56,251 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for b7934444caa228b9e5d6d7b28b59a796 2024-11-17T02:03:56,251 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for b7934444caa228b9e5d6d7b28b59a796 2024-11-17T02:03:56,252 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1731809034900.112f5bbe15f72793585db9ba98950976., pid=42, masterSystemTime=1731809035982 2024-11-17T02:03:56,255 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for b7934444caa228b9e5d6d7b28b59a796 2024-11-17T02:03:56,257 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=13 2024-11-17T02:03:56,257 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,34033,1731809032455 in 376 msec 2024-11-17T02:03:56,258 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/b7934444caa228b9e5d6d7b28b59a796/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T02:03:56,259 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened b7934444caa228b9e5d6d7b28b59a796; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71409085, jitterRate=0.06407828629016876}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T02:03:56,259 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b7934444caa228b9e5d6d7b28b59a796 2024-11-17T02:03:56,259 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for b7934444caa228b9e5d6d7b28b59a796: Running coprocessor pre-open hook at 1731809036239Writing region info on filesystem at 1731809036239Initializing all the Stores at 1731809036241 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731809036241Cleaning up temporary data from old regions at 1731809036251 (+10 ms)Running coprocessor post-open hooks at 1731809036259 (+8 ms)Region opened successfully at 1731809036259 2024-11-17T02:03:56,259 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1731809034900.112f5bbe15f72793585db9ba98950976. 2024-11-17T02:03:56,259 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,fff,1731809034900.112f5bbe15f72793585db9ba98950976. 2024-11-17T02:03:56,260 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,vvv,1731809034900.2f4fe90ec03839f5d26acd9d02569e56. 2024-11-17T02:03:56,260 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1731809034900.b7934444caa228b9e5d6d7b28b59a796., pid=45, masterSystemTime=1731809036000 2024-11-17T02:03:56,260 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN in 665 msec 2024-11-17T02:03:56,260 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 2f4fe90ec03839f5d26acd9d02569e56, NAME => 'test,vvv,1731809034900.2f4fe90ec03839f5d26acd9d02569e56.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-17T02:03:56,260 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2f4fe90ec03839f5d26acd9d02569e56 2024-11-17T02:03:56,261 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,vvv,1731809034900.2f4fe90ec03839f5d26acd9d02569e56.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:56,261 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 2f4fe90ec03839f5d26acd9d02569e56 2024-11-17T02:03:56,261 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 2f4fe90ec03839f5d26acd9d02569e56 2024-11-17T02:03:56,262 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=112f5bbe15f72793585db9ba98950976, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,34033,1731809032455 2024-11-17T02:03:56,263 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1731809034900.b7934444caa228b9e5d6d7b28b59a796. 2024-11-17T02:03:56,263 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,rrr,1731809034900.b7934444caa228b9e5d6d7b28b59a796. 2024-11-17T02:03:56,263 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,ppp,1731809034900.8e1c1fc49350743a9cde50c433c8cb19. 2024-11-17T02:03:56,263 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 8e1c1fc49350743a9cde50c433c8cb19, NAME => 'test,ppp,1731809034900.8e1c1fc49350743a9cde50c433c8cb19.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-17T02:03:56,263 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8e1c1fc49350743a9cde50c433c8cb19 2024-11-17T02:03:56,263 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,ppp,1731809034900.8e1c1fc49350743a9cde50c433c8cb19.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:56,264 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 8e1c1fc49350743a9cde50c433c8cb19 2024-11-17T02:03:56,264 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 8e1c1fc49350743a9cde50c433c8cb19 2024-11-17T02:03:56,263 INFO [StoreOpener-2f4fe90ec03839f5d26acd9d02569e56-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2f4fe90ec03839f5d26acd9d02569e56 2024-11-17T02:03:56,266 INFO [StoreOpener-8e1c1fc49350743a9cde50c433c8cb19-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8e1c1fc49350743a9cde50c433c8cb19 2024-11-17T02:03:56,266 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 112f5bbe15f72793585db9ba98950976, server=c6c0d7a0a7c0,34033,1731809032455 because future has completed 2024-11-17T02:03:56,267 INFO [StoreOpener-2f4fe90ec03839f5d26acd9d02569e56-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2f4fe90ec03839f5d26acd9d02569e56 columnFamilyName family 2024-11-17T02:03:56,267 DEBUG [StoreOpener-2f4fe90ec03839f5d26acd9d02569e56-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:56,268 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=b7934444caa228b9e5d6d7b28b59a796, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:56,268 INFO [StoreOpener-2f4fe90ec03839f5d26acd9d02569e56-1 {}] regionserver.HStore(327): Store=2f4fe90ec03839f5d26acd9d02569e56/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T02:03:56,268 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 2f4fe90ec03839f5d26acd9d02569e56 2024-11-17T02:03:56,269 INFO [StoreOpener-8e1c1fc49350743a9cde50c433c8cb19-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e1c1fc49350743a9cde50c433c8cb19 columnFamilyName family 2024-11-17T02:03:56,269 DEBUG [StoreOpener-8e1c1fc49350743a9cde50c433c8cb19-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:56,270 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/2f4fe90ec03839f5d26acd9d02569e56 2024-11-17T02:03:56,270 INFO [StoreOpener-8e1c1fc49350743a9cde50c433c8cb19-1 {}] regionserver.HStore(327): Store=8e1c1fc49350743a9cde50c433c8cb19/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T02:03:56,270 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/2f4fe90ec03839f5d26acd9d02569e56 2024-11-17T02:03:56,271 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 8e1c1fc49350743a9cde50c433c8cb19 2024-11-17T02:03:56,271 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 2f4fe90ec03839f5d26acd9d02569e56 2024-11-17T02:03:56,271 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7934444caa228b9e5d6d7b28b59a796, server=c6c0d7a0a7c0,33127,1731809032263 because future has completed 2024-11-17T02:03:56,272 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 2f4fe90ec03839f5d26acd9d02569e56 2024-11-17T02:03:56,272 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/8e1c1fc49350743a9cde50c433c8cb19 2024-11-17T02:03:56,273 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/8e1c1fc49350743a9cde50c433c8cb19 2024-11-17T02:03:56,273 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 8e1c1fc49350743a9cde50c433c8cb19 2024-11-17T02:03:56,273 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 8e1c1fc49350743a9cde50c433c8cb19 2024-11-17T02:03:56,274 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 2f4fe90ec03839f5d26acd9d02569e56 2024-11-17T02:03:56,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=10 2024-11-17T02:03:56,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 112f5bbe15f72793585db9ba98950976, server=c6c0d7a0a7c0,34033,1731809032455 in 432 msec 2024-11-17T02:03:56,277 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/2f4fe90ec03839f5d26acd9d02569e56/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T02:03:56,277 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 8e1c1fc49350743a9cde50c433c8cb19 2024-11-17T02:03:56,278 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 2f4fe90ec03839f5d26acd9d02569e56; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68859887, jitterRate=0.026092275977134705}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T02:03:56,278 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2f4fe90ec03839f5d26acd9d02569e56 2024-11-17T02:03:56,278 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 2f4fe90ec03839f5d26acd9d02569e56: Running coprocessor pre-open hook at 1731809036261Writing region info on filesystem at 1731809036261Initializing all the Stores at 1731809036263 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731809036263Cleaning up temporary data from old regions at 1731809036272 (+9 ms)Running coprocessor post-open hooks at 1731809036278 (+6 ms)Region opened successfully at 1731809036278 2024-11-17T02:03:56,279 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1731809034900.2f4fe90ec03839f5d26acd9d02569e56., pid=51, masterSystemTime=1731809035982 2024-11-17T02:03:56,279 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN in 685 msec 2024-11-17T02:03:56,282 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/8e1c1fc49350743a9cde50c433c8cb19/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T02:03:56,282 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1731809034900.2f4fe90ec03839f5d26acd9d02569e56. 2024-11-17T02:03:56,282 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,vvv,1731809034900.2f4fe90ec03839f5d26acd9d02569e56. 2024-11-17T02:03:56,283 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 8e1c1fc49350743a9cde50c433c8cb19; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65177049, jitterRate=-0.02878628671169281}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T02:03:56,283 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8e1c1fc49350743a9cde50c433c8cb19 2024-11-17T02:03:56,283 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 8e1c1fc49350743a9cde50c433c8cb19: Running coprocessor pre-open hook at 1731809036264Writing region info on filesystem at 1731809036264Initializing all the Stores at 1731809036265 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731809036265Cleaning up temporary data from old regions at 1731809036273 (+8 ms)Running coprocessor post-open hooks at 1731809036283 (+10 ms)Region opened successfully at 1731809036283 2024-11-17T02:03:56,284 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1731809034900.8e1c1fc49350743a9cde50c433c8cb19., pid=44, masterSystemTime=1731809036000 2024-11-17T02:03:56,285 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=22 2024-11-17T02:03:56,285 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure b7934444caa228b9e5d6d7b28b59a796, server=c6c0d7a0a7c0,33127,1731809032263 in 424 msec 2024-11-17T02:03:56,287 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1731809034900.8e1c1fc49350743a9cde50c433c8cb19. 2024-11-17T02:03:56,287 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,ppp,1731809034900.8e1c1fc49350743a9cde50c433c8cb19. 2024-11-17T02:03:56,287 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,xxx,1731809034900.9d86e1fe125f78846dcaa5b9921c72db. 2024-11-17T02:03:56,287 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 9d86e1fe125f78846dcaa5b9921c72db, NAME => 'test,xxx,1731809034900.9d86e1fe125f78846dcaa5b9921c72db.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-17T02:03:56,288 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 9d86e1fe125f78846dcaa5b9921c72db 2024-11-17T02:03:56,288 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,xxx,1731809034900.9d86e1fe125f78846dcaa5b9921c72db.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:56,288 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 9d86e1fe125f78846dcaa5b9921c72db 2024-11-17T02:03:56,288 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b7934444caa228b9e5d6d7b28b59a796, ASSIGN in 693 msec 2024-11-17T02:03:56,288 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 9d86e1fe125f78846dcaa5b9921c72db 2024-11-17T02:03:56,290 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=8e1c1fc49350743a9cde50c433c8cb19, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:56,290 INFO [StoreOpener-9d86e1fe125f78846dcaa5b9921c72db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 9d86e1fe125f78846dcaa5b9921c72db 2024-11-17T02:03:56,293 INFO [StoreOpener-9d86e1fe125f78846dcaa5b9921c72db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d86e1fe125f78846dcaa5b9921c72db columnFamilyName family 2024-11-17T02:03:56,293 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8e1c1fc49350743a9cde50c433c8cb19, server=c6c0d7a0a7c0,33127,1731809032263 because future has completed 2024-11-17T02:03:56,293 DEBUG [StoreOpener-9d86e1fe125f78846dcaa5b9921c72db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:56,294 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=2f4fe90ec03839f5d26acd9d02569e56, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,34033,1731809032455 2024-11-17T02:03:56,294 INFO [StoreOpener-9d86e1fe125f78846dcaa5b9921c72db-1 {}] regionserver.HStore(327): Store=9d86e1fe125f78846dcaa5b9921c72db/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T02:03:56,294 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 9d86e1fe125f78846dcaa5b9921c72db 2024-11-17T02:03:56,297 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/9d86e1fe125f78846dcaa5b9921c72db 2024-11-17T02:03:56,298 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/9d86e1fe125f78846dcaa5b9921c72db 2024-11-17T02:03:56,298 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2f4fe90ec03839f5d26acd9d02569e56, server=c6c0d7a0a7c0,34033,1731809032455 because future has completed 2024-11-17T02:03:56,299 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 9d86e1fe125f78846dcaa5b9921c72db 2024-11-17T02:03:56,299 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 9d86e1fe125f78846dcaa5b9921c72db 2024-11-17T02:03:56,301 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 9d86e1fe125f78846dcaa5b9921c72db 2024-11-17T02:03:56,302 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=20 2024-11-17T02:03:56,302 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 8e1c1fc49350743a9cde50c433c8cb19, server=c6c0d7a0a7c0,33127,1731809032263 in 446 msec 2024-11-17T02:03:56,305 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/9d86e1fe125f78846dcaa5b9921c72db/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T02:03:56,305 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8e1c1fc49350743a9cde50c433c8cb19, ASSIGN in 710 msec 2024-11-17T02:03:56,306 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 9d86e1fe125f78846dcaa5b9921c72db; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70260584, jitterRate=0.046964287757873535}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T02:03:56,306 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9d86e1fe125f78846dcaa5b9921c72db 2024-11-17T02:03:56,306 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 9d86e1fe125f78846dcaa5b9921c72db: Running coprocessor pre-open hook at 1731809036288Writing region info on filesystem at 1731809036288Initializing all the Stores at 1731809036290 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731809036290Cleaning up temporary data from old regions at 1731809036299 (+9 ms)Running coprocessor post-open hooks at 1731809036306 (+7 ms)Region opened successfully at 1731809036306 2024-11-17T02:03:56,307 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1731809034900.9d86e1fe125f78846dcaa5b9921c72db., pid=46, masterSystemTime=1731809036000 2024-11-17T02:03:56,310 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=26 2024-11-17T02:03:56,310 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1731809034900.9d86e1fe125f78846dcaa5b9921c72db. 2024-11-17T02:03:56,310 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,xxx,1731809034900.9d86e1fe125f78846dcaa5b9921c72db. 2024-11-17T02:03:56,310 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 2f4fe90ec03839f5d26acd9d02569e56, server=c6c0d7a0a7c0,34033,1731809032455 in 418 msec 2024-11-17T02:03:56,313 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN in 718 msec 2024-11-17T02:03:56,314 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=9d86e1fe125f78846dcaa5b9921c72db, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:56,317 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d86e1fe125f78846dcaa5b9921c72db, server=c6c0d7a0a7c0,33127,1731809032263 because future has completed 2024-11-17T02:03:56,319 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=14 2024-11-17T02:03:56,319 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 05534a4a027a1fdc90227b7a2e681933, server=c6c0d7a0a7c0,33127,1731809032263 in 400 msec 2024-11-17T02:03:56,321 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=05534a4a027a1fdc90227b7a2e681933, ASSIGN in 728 msec 2024-11-17T02:03:56,326 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=28 2024-11-17T02:03:56,326 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure 9d86e1fe125f78846dcaa5b9921c72db, server=c6c0d7a0a7c0,33127,1731809032263 in 459 msec 2024-11-17T02:03:56,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=9d86e1fe125f78846dcaa5b9921c72db, ASSIGN in 735 msec 2024-11-17T02:03:56,331 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=6fd33da09f9d938215ac0e7127ad70d5, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:56,334 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6fd33da09f9d938215ac0e7127ad70d5, server=c6c0d7a0a7c0,45037,1731809032394 because future has completed 2024-11-17T02:03:56,336 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=11 2024-11-17T02:03:56,337 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 7517bce9e04879764302cccd45af20fb, server=c6c0d7a0a7c0,33127,1731809032263 in 421 msec 2024-11-17T02:03:56,339 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7517bce9e04879764302cccd45af20fb, ASSIGN in 745 msec 2024-11-17T02:03:56,343 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=12 2024-11-17T02:03:56,343 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 6fd33da09f9d938215ac0e7127ad70d5, server=c6c0d7a0a7c0,45037,1731809032394 in 552 msec 2024-11-17T02:03:56,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6fd33da09f9d938215ac0e7127ad70d5, ASSIGN in 751 msec 2024-11-17T02:03:56,346 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=81f322eb57dcb42c3b4269a738bafd6d, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:56,350 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=037b7a93730957425bc54697b3594bb2, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:56,350 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81f322eb57dcb42c3b4269a738bafd6d, server=c6c0d7a0a7c0,33127,1731809032263 because future has completed 2024-11-17T02:03:56,355 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 037b7a93730957425bc54697b3594bb2, server=c6c0d7a0a7c0,45037,1731809032394 because future has completed 2024-11-17T02:03:56,356 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32803 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=c6c0d7a0a7c0,33127,1731809032263, table=test, region=81f322eb57dcb42c3b4269a738bafd6d. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-17T02:03:56,357 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32803 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=c6c0d7a0a7c0,45037,1731809032394, table=test, region=037b7a93730957425bc54697b3594bb2. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-17T02:03:56,360 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=7 2024-11-17T02:03:56,360 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 81f322eb57dcb42c3b4269a738bafd6d, server=c6c0d7a0a7c0,33127,1731809032263 in 448 msec 2024-11-17T02:03:56,362 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=81f322eb57dcb42c3b4269a738bafd6d, ASSIGN in 769 msec 2024-11-17T02:03:56,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-17T02:03:56,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure 037b7a93730957425bc54697b3594bb2, server=c6c0d7a0a7c0,45037,1731809032394 in 595 msec 2024-11-17T02:03:56,368 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=037b7a93730957425bc54697b3594bb2, ASSIGN in 774 msec 2024-11-17T02:03:56,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN in 776 msec 2024-11-17T02:03:56,371 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=77bd97fde6bd169f3e20553b45a65e2c, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:56,374 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 77bd97fde6bd169f3e20553b45a65e2c, server=c6c0d7a0a7c0,33127,1731809032263 because future has completed 2024-11-17T02:03:56,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=25 2024-11-17T02:03:56,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 77bd97fde6bd169f3e20553b45a65e2c, server=c6c0d7a0a7c0,33127,1731809032263 in 529 msec 2024-11-17T02:03:56,382 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=4 2024-11-17T02:03:56,382 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=77bd97fde6bd169f3e20553b45a65e2c, ASSIGN in 787 msec 2024-11-17T02:03:56,383 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T02:03:56,384 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731809036383"}]},"ts":"1731809036383"} 2024-11-17T02:03:56,387 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-11-17T02:03:56,388 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T02:03:56,394 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 1.4810 sec 2024-11-17T02:03:57,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T02:03:57,072 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-11-17T02:03:57,073 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-11-17T02:03:57,075 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T02:03:57,096 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-11-17T02:03:57,097 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T02:03:57,098 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-11-17T02:03:57,098 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T02:03:57,105 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-17T02:03:57,134 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-17T02:03:57,152 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T02:03:57,155 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38510, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T02:03:57,158 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T02:03:57,161 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48408, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T02:03:57,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 81f322eb57dcb42c3b4269a738bafd6d changed from -1.0 to 0.0, refreshing cache 2024-11-17T02:03:57,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8356c0fdb856c376b8678f2c2597565a changed from -1.0 to 0.0, refreshing cache 2024-11-17T02:03:57,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 7517bce9e04879764302cccd45af20fb changed from -1.0 to 0.0, refreshing cache 2024-11-17T02:03:57,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 05534a4a027a1fdc90227b7a2e681933 changed from -1.0 to 0.0, refreshing cache 2024-11-17T02:03:57,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 663bac36df6c9873ad776bb3ad42409f changed from -1.0 to 0.0, refreshing cache 2024-11-17T02:03:57,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8e1c1fc49350743a9cde50c433c8cb19 changed from -1.0 to 0.0, refreshing cache 2024-11-17T02:03:57,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b7934444caa228b9e5d6d7b28b59a796 changed from -1.0 to 0.0, refreshing cache 2024-11-17T02:03:57,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 77bd97fde6bd169f3e20553b45a65e2c changed from -1.0 to 0.0, refreshing cache 2024-11-17T02:03:57,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 9d86e1fe125f78846dcaa5b9921c72db changed from -1.0 to 0.0, refreshing cache 2024-11-17T02:03:57,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region de9924be32211cddceef43e5b4866dc8 changed from -1.0 to 0.0, refreshing cache 2024-11-17T02:03:57,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 112f5bbe15f72793585db9ba98950976 changed from -1.0 to 0.0, refreshing cache 2024-11-17T02:03:57,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 03b7adb3afa9bebd86dc1da38125b982 changed from -1.0 to 0.0, refreshing cache 2024-11-17T02:03:57,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1056118c93716b24d18f749826cf316a changed from -1.0 to 0.0, refreshing cache 2024-11-17T02:03:57,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 25a67f85380b4a3cd6890ab81d0b3a27 changed from -1.0 to 0.0, refreshing cache 2024-11-17T02:03:57,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b6eba00b85aa16d5ccbfc8e3c35ff298 changed from -1.0 to 0.0, refreshing cache 2024-11-17T02:03:57,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2f4fe90ec03839f5d26acd9d02569e56 changed from -1.0 to 0.0, refreshing cache 2024-11-17T02:03:57,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 7eed07a820ab15d7af64462e2c1b70f7 changed from -1.0 to 0.0, refreshing cache 2024-11-17T02:03:57,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a17ac9e270344fa2752f51f8196693b2 changed from -1.0 to 0.0, refreshing cache 2024-11-17T02:03:57,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 15c64fab21a3ca03ec0a7afabc19cdc9 changed from -1.0 to 0.0, refreshing cache 2024-11-17T02:03:57,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6fd33da09f9d938215ac0e7127ad70d5 changed from -1.0 to 0.0, refreshing cache 2024-11-17T02:03:57,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ac10d67ff6c3f527cc0861e738032bdb changed from -1.0 to 0.0, refreshing cache 2024-11-17T02:03:57,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 74108f0b014075d4993f0eee396e39e5 changed from -1.0 to 0.0, refreshing cache 2024-11-17T02:03:57,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 899f31eacba6d2d0aedbee7d8bbb4c9d changed from -1.0 to 0.0, refreshing cache 2024-11-17T02:03:57,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4748cc4c5ad3838a3d0568cee797c169 changed from -1.0 to 0.0, refreshing cache 2024-11-17T02:03:57,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region db157dde5b6770afbf869f7733d95efb changed from -1.0 to 0.0, refreshing cache 2024-11-17T02:03:57,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 037b7a93730957425bc54697b3594bb2 changed from -1.0 to 0.0, refreshing cache 2024-11-17T02:03:57,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-11-17T02:03:57,225 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-11-17T02:03:57,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-11-17T02:03:57,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] balancer.BalancerClusterState(204): Hosts are {c6c0d7a0a7c0=0} racks are {/default-rack=0} 2024-11-17T02:03:57,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] balancer.BalancerClusterState(303): server 0 has 9 regions 2024-11-17T02:03:57,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] balancer.BalancerClusterState(303): server 1 has 9 regions 2024-11-17T02:03:57,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] balancer.BalancerClusterState(303): server 2 has 9 regions 2024-11-17T02:03:57,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-17T02:03:57,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-17T02:03:57,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-17T02:03:57,226 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-17T02:03:57,227 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-17T02:03:57,227 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-17T02:03:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-11-17T02:03:57,232 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] balancer.StochasticLoadBalancer(395): Cluster wide - skipping load balancing because weighted average imbalance=0.016474464579901153 <= threshold(0.025). If you want more aggressive balancing, either lower hbase.master.balancer.stochastic.minCostNeedBalance from 0.025 or increase the relative multiplier(s) of the specific cost function(s). functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-11-17T02:03:57,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-11-17T02:03:57,234 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T02:03:57,244 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.2 set balanceSwitch=false 2024-11-17T02:03:57,263 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=384, OpenFileDescriptor=649, MaxFileDescriptor=1048576, SystemLoadAverage=122, ProcessCount=11, AvailableMemoryMB=5775 2024-11-17T02:03:57,267 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-11-17T02:03:57,268 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-11-17T02:03:57,268 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing c6c0d7a0a7c0,34033,1731809032455 2024-11-17T02:03:57,269 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server c6c0d7a0a7c0,34033,1731809032455: Simulated kill ***** 2024-11-17T02:03:57,269 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-17T02:03:57,271 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-17T02:03:57,274 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-17T02:03:57,274 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-17T02:03:57,275 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-17T02:03:57,275 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 309046512 }, "NonHeapMemoryUsage": { "committed": 151519232, "init": 7667712, "max": -1, "used": 149167432 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "c6c0d7a0a7c0" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "c6c0d7a0a7c0" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "c6c0d7a0a7c0" } ] } 2024-11-17T02:03:57,278 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32803 {}] master.MasterRpcServices(700): c6c0d7a0a7c0,34033,1731809032455 reported a fatal error: ***** ABORTING region server c6c0d7a0a7c0,34033,1731809032455: Simulated kill ***** 2024-11-17T02:03:57,281 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c6c0d7a0a7c0,34033,1731809032455' ***** 2024-11-17T02:03:57,281 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-17T02:03:57,281 INFO [RS:2;c6c0d7a0a7c0:34033 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-17T02:03:57,282 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-17T02:03:57,282 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T02:03:57,282 INFO [RS:2;c6c0d7a0a7c0:34033 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-17T02:03:57,282 INFO [RS:2;c6c0d7a0a7c0:34033 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-17T02:03:57,282 INFO [RS:2;c6c0d7a0a7c0:34033 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-17T02:03:57,282 INFO [RS:2;c6c0d7a0a7c0:34033 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;c6c0d7a0a7c0:34033. 2024-11-17T02:03:57,282 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T02:03:57,287 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T02:03:57,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741836_1012 (size=32114) 2024-11-17T02:03:57,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741836_1012 (size=32114) 2024-11-17T02:03:57,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741836_1012 (size=32114) 2024-11-17T02:03:57,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741835_1011 (size=2232) 2024-11-17T02:03:57,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741835_1011 (size=2232) 2024-11-17T02:03:57,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741835_1011 (size=2232) 2024-11-17T02:03:57,300 DEBUG [RS:2;c6c0d7a0a7c0:34033 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T02:03:57,300 INFO [RS:2;c6c0d7a0a7c0:34033 {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T02:03:57,300 INFO [RS:2;c6c0d7a0a7c0:34033 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34033 2024-11-17T02:03:57,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34033-0x10147d245010003, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c6c0d7a0a7c0,34033,1731809032455 2024-11-17T02:03:57,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-17T02:03:57,361 INFO [RS:2;c6c0d7a0a7c0:34033 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-17T02:03:57,365 INFO [regionserver/c6c0d7a0a7c0:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T02:03:57,370 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c6c0d7a0a7c0,34033,1731809032455] 2024-11-17T02:03:57,378 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c6c0d7a0a7c0,34033,1731809032455 already deleted, retry=false 2024-11-17T02:03:57,378 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of c6c0d7a0a7c0,34033,1731809032455 on c6c0d7a0a7c0,32803,1731809031530 2024-11-17T02:03:57,384 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true 2024-11-17T02:03:57,390 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=57 for c6c0d7a0a7c0,34033,1731809032455 (carryingMeta=true) c6c0d7a0a7c0,34033,1731809032455/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@160df6ba[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-17T02:03:57,390 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true 2024-11-17T02:03:57,392 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true, isMeta: true 2024-11-17T02:03:57,394 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,34033,1731809032455-splitting 2024-11-17T02:03:57,395 INFO [PEWorker-1 {}] master.SplitWALManager(105): c6c0d7a0a7c0,34033,1731809032455 WAL count=1, meta=true 2024-11-17T02:03:57,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta}] 2024-11-17T02:03:57,402 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:57,405 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263}] 2024-11-17T02:03:57,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34033-0x10147d245010003, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T02:03:57,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34033-0x10147d245010003, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T02:03:57,470 INFO [RS:2;c6c0d7a0a7c0:34033 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-17T02:03:57,470 INFO [RS:2;c6c0d7a0a7c0:34033 {}] regionserver.HRegionServer(1031): Exiting; stopping=c6c0d7a0a7c0,34033,1731809032455; zookeeper connection closed. 2024-11-17T02:03:57,471 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5adaf09f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5adaf09f 2024-11-17T02:03:57,483 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T02:03:57,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33127 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=59 2024-11-17T02:03:57,592 INFO [RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(299): Splitting hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,34033,1731809032455-splitting/c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, size=31.4 K (32114bytes) 2024-11-17T02:03:57,593 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-17T02:03:57,593 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-17T02:03:57,593 INFO [RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,34033,1731809032455-splitting/c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta 2024-11-17T02:03:57,598 INFO [RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,34033,1731809032455-splitting/c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta after 3ms 2024-11-17T02:03:57,605 INFO [RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(310): Open hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,34033,1731809032455-splitting/c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta took 14ms 2024-11-17T02:03:57,615 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-17T02:03:57,623 INFO [RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta.temp 2024-11-17T02:03:57,626 INFO [RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta.temp 2024-11-17T02:03:57,627 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,34033,1731809032455-splitting/c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta so closing down 2024-11-17T02:03:57,628 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-17T02:03:57,629 INFO [RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(145): 3 split writer threads finished 2024-11-17T02:03:57,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741864_1040 (size=31873) 2024-11-17T02:03:57,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741864_1040 (size=31873) 2024-11-17T02:03:57,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741864_1040 (size=31873) 2024-11-17T02:03:57,640 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta.temp (wrote 57 edits, skipped 0 edits in 2 ms) 2024-11-17T02:03:57,643 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta.temp to hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-17T02:03:57,646 INFO [RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(425): Processed 58 edits across 1 Regions in 36 ms; skipped=1; WAL=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,34033,1731809032455-splitting/c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false 2024-11-17T02:03:57,646 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(428): Completed split of hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,34033,1731809032455-splitting/c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, journal: Splitting hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,34033,1731809032455-splitting/c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, size=31.4 K (32114bytes) at 1731809037592Creating recovered edits writer path=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta.temp at 1731809037626 (+34 ms)Finishing writing output for hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,34033,1731809032455-splitting/c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta so closing down at 1731809037627 (+1 ms)3 split writer threads finished at 1731809037629 (+2 ms)Closed recovered edits writer path=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta.temp (wrote 57 edits, skipped 0 edits in 2 ms) at 1731809037640 (+11 ms)Rename recovered edits hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta.temp to hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 at 1731809037643 (+3 ms)Processed 58 edits across 1 Regions in 36 ms; skipped=1; WAL=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,34033,1731809032455-splitting/c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false at 1731809037646 (+3 ms) 2024-11-17T02:03:57,646 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,34033,1731809032455-splitting/c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta 2024-11-17T02:03:57,648 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-17T02:03:57,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-17T02:03:57,658 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,34033,1731809032455-splitting/c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta to hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/oldWALs 2024-11-17T02:03:57,661 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-17T02:03:57,661 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 in 253 msec 2024-11-17T02:03:57,663 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:57,666 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-17T02:03:57,666 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 in 266 msec 2024-11-17T02:03:57,669 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,34033,1731809032455-splitting dir is empty, no logs to split. 2024-11-17T02:03:57,669 INFO [PEWorker-1 {}] master.SplitWALManager(105): c6c0d7a0a7c0,34033,1731809032455 WAL count=0, meta=true 2024-11-17T02:03:57,669 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if c6c0d7a0a7c0,34033,1731809032455 WAL splitting is done? wals=0, meta=true 2024-11-17T02:03:57,672 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-17T02:03:57,674 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T02:03:57,675 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-17T02:03:57,825 DEBUG [c6c0d7a0a7c0:32803 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-17T02:03:57,826 DEBUG [c6c0d7a0a7c0:32803 {}] balancer.BalancerClusterState(204): Hosts are {c6c0d7a0a7c0=0} racks are {/default-rack=0} 2024-11-17T02:03:57,826 DEBUG [c6c0d7a0a7c0:32803 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-17T02:03:57,826 DEBUG [c6c0d7a0a7c0:32803 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-17T02:03:57,826 DEBUG [c6c0d7a0a7c0:32803 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-17T02:03:57,826 DEBUG [c6c0d7a0a7c0:32803 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-17T02:03:57,826 INFO [c6c0d7a0a7c0:32803 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-17T02:03:57,826 INFO [c6c0d7a0a7c0:32803 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-17T02:03:57,826 DEBUG [c6c0d7a0a7c0:32803 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-17T02:03:57,827 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:57,830 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c6c0d7a0a7c0,45037,1731809032394, state=OPENING 2024-11-17T02:03:57,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:57,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:57,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:57,869 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T02:03:57,870 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:57,870 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,45037,1731809032394}] 2024-11-17T02:03:57,870 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:57,870 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:58,000 INFO [regionserver/c6c0d7a0a7c0:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-17T02:03:58,000 INFO [regionserver/c6c0d7a0a7c0:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-17T02:03:58,029 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-17T02:03:58,029 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T02:03:58,029 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-17T02:03:58,032 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c6c0d7a0a7c0%2C45037%2C1731809032394.meta, suffix=.meta, logDir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,45037,1731809032394, archiveDir=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/oldWALs, maxLogs=32 2024-11-17T02:03:58,048 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,45037,1731809032394/c6c0d7a0a7c0%2C45037%2C1731809032394.meta.1731809038035.meta, exclude list is [], retry=0 2024-11-17T02:03:58,051 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33655,DS-19b795a5-ad52-4253-ba85-390185288d22,DISK] 2024-11-17T02:03:58,051 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36067,DS-11c5e779-9b67-42ae-934a-e223a96170e4,DISK] 2024-11-17T02:03:58,051 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36455,DS-da656d4a-0400-49ad-97b4-848c04dabe50,DISK] 2024-11-17T02:03:58,053 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,45037,1731809032394/c6c0d7a0a7c0%2C45037%2C1731809032394.meta.1731809038035.meta 2024-11-17T02:03:58,054 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40369:40369),(127.0.0.1/127.0.0.1:40831:40831),(127.0.0.1/127.0.0.1:37075:37075)] 2024-11-17T02:03:58,054 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-17T02:03:58,054 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-17T02:03:58,054 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-17T02:03:58,055 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-17T02:03:58,055 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-17T02:03:58,055 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:58,055 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-17T02:03:58,055 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-17T02:03:58,059 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-17T02:03:58,060 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-17T02:03:58,060 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:58,061 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T02:03:58,061 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-17T02:03:58,062 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-17T02:03:58,062 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:58,063 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T02:03:58,063 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-17T02:03:58,064 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-17T02:03:58,064 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:58,064 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T02:03:58,064 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-17T02:03:58,066 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-17T02:03:58,066 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:58,067 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T02:03:58,067 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-17T02:03:58,068 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/hbase/meta/1588230740 2024-11-17T02:03:58,070 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/hbase/meta/1588230740 2024-11-17T02:03:58,071 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-17T02:03:58,087 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5793): Applied 264, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=60, path=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-17T02:03:58,091 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=28 KB heapSize=48.31 KB 2024-11-17T02:03:58,158 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/hbase/meta/1588230740/.tmp/info/e0d453c0794e4eaea242a562ade717cc is 138, key is test,bbb,1731809034900.a17ac9e270344fa2752f51f8196693b2./info:regioninfo/1731809035954/Put/seqid=0 2024-11-17T02:03:58,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741866_1042 (size=34924) 2024-11-17T02:03:58,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741866_1042 (size=34924) 2024-11-17T02:03:58,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741866_1042 (size=34924) 2024-11-17T02:03:58,178 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.86 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/hbase/meta/1588230740/.tmp/info/e0d453c0794e4eaea242a562ade717cc 2024-11-17T02:03:58,236 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/hbase/meta/1588230740/.tmp/ns/2d8feae2fc4a44b3855bab3ae6573172 is 43, key is default/ns:d/1731809034599/Put/seqid=0 2024-11-17T02:03:58,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741867_1043 (size=5153) 2024-11-17T02:03:58,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741867_1043 (size=5153) 2024-11-17T02:03:58,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741867_1043 (size=5153) 2024-11-17T02:03:58,247 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/hbase/meta/1588230740/.tmp/ns/2d8feae2fc4a44b3855bab3ae6573172 2024-11-17T02:03:58,272 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/hbase/meta/1588230740/.tmp/table/09849fb22a1a42c6a2e1aee048fc8897 is 40, key is test/table:state/1731809036383/Put/seqid=0 2024-11-17T02:03:58,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741868_1044 (size=5165) 2024-11-17T02:03:58,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741868_1044 (size=5165) 2024-11-17T02:03:58,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741868_1044 (size=5165) 2024-11-17T02:03:58,281 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/hbase/meta/1588230740/.tmp/table/09849fb22a1a42c6a2e1aee048fc8897 2024-11-17T02:03:58,290 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/hbase/meta/1588230740/.tmp/info/e0d453c0794e4eaea242a562ade717cc as hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/hbase/meta/1588230740/info/e0d453c0794e4eaea242a562ade717cc 2024-11-17T02:03:58,300 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/hbase/meta/1588230740/info/e0d453c0794e4eaea242a562ade717cc, entries=260, sequenceid=60, filesize=34.1 K 2024-11-17T02:03:58,303 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/hbase/meta/1588230740/.tmp/ns/2d8feae2fc4a44b3855bab3ae6573172 as hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/hbase/meta/1588230740/ns/2d8feae2fc4a44b3855bab3ae6573172 2024-11-17T02:03:58,313 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/hbase/meta/1588230740/ns/2d8feae2fc4a44b3855bab3ae6573172, entries=2, sequenceid=60, filesize=5.0 K 2024-11-17T02:03:58,315 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/hbase/meta/1588230740/.tmp/table/09849fb22a1a42c6a2e1aee048fc8897 as hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/hbase/meta/1588230740/table/09849fb22a1a42c6a2e1aee048fc8897 2024-11-17T02:03:58,324 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/hbase/meta/1588230740/table/09849fb22a1a42c6a2e1aee048fc8897, entries=2, sequenceid=60, filesize=5.0 K 2024-11-17T02:03:58,329 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 236ms, sequenceid=60, compaction requested=false; wal=null 2024-11-17T02:03:58,330 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-17T02:03:58,334 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-17T02:03:58,335 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-17T02:03:58,335 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-17T02:03:58,336 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-17T02:03:58,338 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-17T02:03:58,341 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/hbase/meta/1588230740/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=1 2024-11-17T02:03:58,342 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=61; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70968087, jitterRate=0.057506904006004333}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-17T02:03:58,342 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-17T02:03:58,343 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731809038055Writing region info on filesystem at 1731809038055Initializing all the Stores at 1731809038057 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731809038057Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731809038058 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731809038058Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731809038058Obtaining lock to block concurrent updates at 1731809038091 (+33 ms)Preparing flush snapshotting stores in 1588230740 at 1731809038091Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=28672, getHeapSize=49408, getOffHeapSize=0, getCellsCount=264 at 1731809038094 (+3 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731809038094Flushing 1588230740/info: creating writer at 1731809038095 (+1 ms)Flushing 1588230740/info: appending metadata at 1731809038147 (+52 ms)Flushing 1588230740/info: closing flushed file at 1731809038150 (+3 ms)Flushing 1588230740/ns: creating writer at 1731809038221 (+71 ms)Flushing 1588230740/ns: appending metadata at 1731809038235 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1731809038235Flushing 1588230740/table: creating writer at 1731809038256 (+21 ms)Flushing 1588230740/table: appending metadata at 1731809038271 (+15 ms)Flushing 1588230740/table: closing flushed file at 1731809038271Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@201c8c92: reopening flushed file at 1731809038289 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@173c4c8b: reopening flushed file at 1731809038302 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@135b6463: reopening flushed file at 1731809038314 (+12 ms)Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 236ms, sequenceid=60, compaction requested=false; wal=null at 1731809038329 (+15 ms)Cleaning up temporary data from old regions at 1731809038335 (+6 ms)Running coprocessor post-open hooks at 1731809038342 (+7 ms)Region opened successfully at 1731809038342 2024-11-17T02:03:58,345 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=61, masterSystemTime=1731809038023 2024-11-17T02:03:58,347 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-17T02:03:58,347 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-17T02:03:58,348 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=61, regionLocation=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:58,349 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c6c0d7a0a7c0,45037,1731809032394, state=OPEN 2024-11-17T02:03:58,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:58,402 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:58,402 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:58,403 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=61, ppid=60, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:58,403 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:58,403 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:58,403 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:58,406 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-17T02:03:58,406 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,45037,1731809032394 in 534 msec 2024-11-17T02:03:58,408 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=57 2024-11-17T02:03:58,409 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 734 msec 2024-11-17T02:03:58,409 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): c6c0d7a0a7c0,34033,1731809032455 had 9 regions 2024-11-17T02:03:58,410 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true, isMeta: false 2024-11-17T02:03:58,412 INFO [PEWorker-1 {}] master.SplitWALManager(105): c6c0d7a0a7c0,34033,1731809032455 WAL count=1, meta=false 2024-11-17T02:03:58,413 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116}] 2024-11-17T02:03:58,415 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:58,416 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394}] 2024-11-17T02:03:58,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45037 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=63 2024-11-17T02:03:58,584 INFO [RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(299): Splitting hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,34033,1731809032455-splitting/c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, size=2.2 K (2232bytes) 2024-11-17T02:03:58,584 INFO [RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,34033,1731809032455-splitting/c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116 2024-11-17T02:03:58,585 INFO [RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,34033,1731809032455-splitting/c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116 after 1ms 2024-11-17T02:03:58,588 INFO [RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(310): Open hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,34033,1731809032455-splitting/c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116 took 4ms 2024-11-17T02:03:58,591 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for b6eba00b85aa16d5ccbfc8e3c35ff298: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-17T02:03:58,592 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 1056118c93716b24d18f749826cf316a: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-17T02:03:58,594 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for de9924be32211cddceef43e5b4866dc8: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-17T02:03:58,596 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 25a67f85380b4a3cd6890ab81d0b3a27: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-17T02:03:58,598 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 7eed07a820ab15d7af64462e2c1b70f7: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-17T02:03:58,601 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 03b7adb3afa9bebd86dc1da38125b982: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-17T02:03:58,603 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 112f5bbe15f72793585db9ba98950976: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-17T02:03:58,605 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 2f4fe90ec03839f5d26acd9d02569e56: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-17T02:03:58,605 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,34033,1731809032455-splitting/c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116 so closing down 2024-11-17T02:03:58,605 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-17T02:03:58,605 INFO [RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(145): 3 split writer threads finished 2024-11-17T02:03:58,606 INFO [RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(425): Processed 8 edits across 0 Regions in 18 ms; skipped=8; WAL=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,34033,1731809032455-splitting/c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, size=2.2 K, length=2232, corrupted=false, cancelled=false 2024-11-17T02:03:58,606 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(428): Completed split of hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,34033,1731809032455-splitting/c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, journal: Splitting hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,34033,1731809032455-splitting/c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, size=2.2 K (2232bytes) at 1731809038584Finishing writing output for hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,34033,1731809032455-splitting/c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116 so closing down at 1731809038605 (+21 ms)3 split writer threads finished at 1731809038605Processed 8 edits across 0 Regions in 18 ms; skipped=8; WAL=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,34033,1731809032455-splitting/c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, size=2.2 K, length=2232, corrupted=false, cancelled=false at 1731809038606 (+1 ms) 2024-11-17T02:03:58,606 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,34033,1731809032455-splitting/c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116 2024-11-17T02:03:58,606 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-17T02:03:58,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32803 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-17T02:03:58,610 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,34033,1731809032455-splitting/c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116 to hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/oldWALs 2024-11-17T02:03:58,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-17T02:03:58,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 in 194 msec 2024-11-17T02:03:58,614 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:58,617 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=57 2024-11-17T02:03:58,617 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 in 201 msec 2024-11-17T02:03:58,619 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,34033,1731809032455-splitting dir is empty, no logs to split. 2024-11-17T02:03:58,619 INFO [PEWorker-1 {}] master.SplitWALManager(105): c6c0d7a0a7c0,34033,1731809032455 WAL count=0, meta=false 2024-11-17T02:03:58,619 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if c6c0d7a0a7c0,34033,1731809032455 WAL splitting is done? wals=0, meta=false 2024-11-17T02:03:58,622 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN}] 2024-11-17T02:03:58,624 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN 2024-11-17T02:03:58,625 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-17T02:03:58,626 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN 2024-11-17T02:03:58,627 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-17T02:03:58,628 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN 2024-11-17T02:03:58,629 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-17T02:03:58,630 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN 2024-11-17T02:03:58,631 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-17T02:03:58,632 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN 2024-11-17T02:03:58,633 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-17T02:03:58,634 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN 2024-11-17T02:03:58,634 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-17T02:03:58,635 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN 2024-11-17T02:03:58,636 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-17T02:03:58,637 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN 2024-11-17T02:03:58,639 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-17T02:03:58,776 DEBUG [c6c0d7a0a7c0:32803 {}] balancer.BalancerClusterState(204): Hosts are {c6c0d7a0a7c0=0} racks are {/default-rack=0} 2024-11-17T02:03:58,776 DEBUG [c6c0d7a0a7c0:32803 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-17T02:03:58,776 DEBUG [c6c0d7a0a7c0:32803 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-17T02:03:58,776 DEBUG [c6c0d7a0a7c0:32803 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-17T02:03:58,776 DEBUG [c6c0d7a0a7c0:32803 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-17T02:03:58,776 INFO [c6c0d7a0a7c0:32803 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-17T02:03:58,776 INFO [c6c0d7a0a7c0:32803 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-17T02:03:58,776 DEBUG [c6c0d7a0a7c0:32803 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-17T02:03:58,777 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=2f4fe90ec03839f5d26acd9d02569e56, regionState=OPENING, regionLocation=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:58,777 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=25a67f85380b4a3cd6890ab81d0b3a27, regionState=OPENING, regionLocation=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:58,777 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=03b7adb3afa9bebd86dc1da38125b982, regionState=OPENING, regionLocation=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:58,778 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=de9924be32211cddceef43e5b4866dc8, regionState=OPENING, regionLocation=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:58,778 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server c6c0d7a0a7c0:34033 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: c6c0d7a0a7c0/172.17.0.2:34033 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T02:03:58,778 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=7eed07a820ab15d7af64462e2c1b70f7, regionState=OPENING, regionLocation=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:58,779 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=b6eba00b85aa16d5ccbfc8e3c35ff298, regionState=OPENING, regionLocation=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:58,779 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=1056118c93716b24d18f749826cf316a, regionState=OPENING, regionLocation=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:58,779 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=112f5bbe15f72793585db9ba98950976, regionState=OPENING, regionLocation=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:58,780 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,34033,1731809032455, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,34033,1731809032455, seqNum=-1, error=java.net.ConnectException: Call to address=c6c0d7a0a7c0:34033 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: c6c0d7a0a7c0/172.17.0.2:34033 2024-11-17T02:03:58,781 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,34033,1731809032455, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-17T02:03:58,781 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,34033,1731809032455, seqNum=-1 from cache 2024-11-17T02:03:58,781 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,34033,1731809032455, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=c6c0d7a0a7c0:34033 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: c6c0d7a0a7c0/172.17.0.2:34033 2024-11-17T02:03:58,782 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address c6c0d7a0a7c0:34033 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: c6c0d7a0a7c0/172.17.0.2:34033 2024-11-17T02:03:58,782 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server c6c0d7a0a7c0:34033 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: c6c0d7a0a7c0/172.17.0.2:34033 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T02:03:58,783 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,34033,1731809032455, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=c6c0d7a0a7c0:34033 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: c6c0d7a0a7c0/172.17.0.2:34033 2024-11-17T02:03:58,783 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,34033,1731809032455, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=c6c0d7a0a7c0:34033 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: c6c0d7a0a7c0/172.17.0.2:34033 2024-11-17T02:03:58,783 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,34033,1731809032455, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=c6c0d7a0a7c0:34033 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: c6c0d7a0a7c0/172.17.0.2:34033 2024-11-17T02:03:58,783 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,34033,1731809032455, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=c6c0d7a0a7c0:34033 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: c6c0d7a0a7c0/172.17.0.2:34033 2024-11-17T02:03:58,783 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,34033,1731809032455, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=c6c0d7a0a7c0:34033 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: c6c0d7a0a7c0/172.17.0.2:34033 2024-11-17T02:03:58,783 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,34033,1731809032455, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=c6c0d7a0a7c0:34033 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: c6c0d7a0a7c0/172.17.0.2:34033 2024-11-17T02:03:58,783 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address c6c0d7a0a7c0:34033 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: c6c0d7a0a7c0/172.17.0.2:34033 2024-11-17T02:03:58,890 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T02:03:58,890 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,45037,1731809032394, seqNum=-1] 2024-11-17T02:03:58,890 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T02:03:58,892 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53227, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T02:03:58,894 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN because future has completed 2024-11-17T02:03:58,895 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2f4fe90ec03839f5d26acd9d02569e56, server=c6c0d7a0a7c0,33127,1731809032263}] 2024-11-17T02:03:58,895 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN because future has completed 2024-11-17T02:03:58,897 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN because future has completed 2024-11-17T02:03:58,898 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN because future has completed 2024-11-17T02:03:58,899 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure de9924be32211cddceef43e5b4866dc8, server=c6c0d7a0a7c0,33127,1731809032263}] 2024-11-17T02:03:58,903 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263}] 2024-11-17T02:03:58,906 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 25a67f85380b4a3cd6890ab81d0b3a27, server=c6c0d7a0a7c0,33127,1731809032263}] 2024-11-17T02:03:58,934 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN because future has completed 2024-11-17T02:03:58,935 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7eed07a820ab15d7af64462e2c1b70f7, server=c6c0d7a0a7c0,45037,1731809032394}] 2024-11-17T02:03:58,936 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN because future has completed 2024-11-17T02:03:58,937 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN because future has completed 2024-11-17T02:03:58,938 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN because future has completed 2024-11-17T02:03:58,939 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 112f5bbe15f72793585db9ba98950976, server=c6c0d7a0a7c0,45037,1731809032394}] 2024-11-17T02:03:58,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1056118c93716b24d18f749826cf316a, server=c6c0d7a0a7c0,45037,1731809032394}] 2024-11-17T02:03:58,945 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure b6eba00b85aa16d5ccbfc8e3c35ff298, server=c6c0d7a0a7c0,45037,1731809032394}] 2024-11-17T02:03:59,055 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(132): Open test,iii,1731809034900.03b7adb3afa9bebd86dc1da38125b982. 2024-11-17T02:03:59,056 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7752): Opening region: {ENCODED => 03b7adb3afa9bebd86dc1da38125b982, NAME => 'test,iii,1731809034900.03b7adb3afa9bebd86dc1da38125b982.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-17T02:03:59,056 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 03b7adb3afa9bebd86dc1da38125b982 2024-11-17T02:03:59,056 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(898): Instantiated test,iii,1731809034900.03b7adb3afa9bebd86dc1da38125b982.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:59,057 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7794): checking encryption for 03b7adb3afa9bebd86dc1da38125b982 2024-11-17T02:03:59,057 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7797): checking classloading for 03b7adb3afa9bebd86dc1da38125b982 2024-11-17T02:03:59,059 INFO [StoreOpener-03b7adb3afa9bebd86dc1da38125b982-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 03b7adb3afa9bebd86dc1da38125b982 2024-11-17T02:03:59,061 INFO [StoreOpener-03b7adb3afa9bebd86dc1da38125b982-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 03b7adb3afa9bebd86dc1da38125b982 columnFamilyName family 2024-11-17T02:03:59,061 DEBUG [StoreOpener-03b7adb3afa9bebd86dc1da38125b982-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:59,061 INFO [StoreOpener-03b7adb3afa9bebd86dc1da38125b982-1 {}] regionserver.HStore(327): Store=03b7adb3afa9bebd86dc1da38125b982/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T02:03:59,062 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1038): replaying wal for 03b7adb3afa9bebd86dc1da38125b982 2024-11-17T02:03:59,063 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/03b7adb3afa9bebd86dc1da38125b982 2024-11-17T02:03:59,065 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/03b7adb3afa9bebd86dc1da38125b982 2024-11-17T02:03:59,066 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1048): stopping wal replay for 03b7adb3afa9bebd86dc1da38125b982 2024-11-17T02:03:59,066 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1060): Cleaning up temporary data for 03b7adb3afa9bebd86dc1da38125b982 2024-11-17T02:03:59,071 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1093): writing seq id for 03b7adb3afa9bebd86dc1da38125b982 2024-11-17T02:03:59,072 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1114): Opened 03b7adb3afa9bebd86dc1da38125b982; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61770569, jitterRate=-0.07954679429531097}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T02:03:59,072 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 03b7adb3afa9bebd86dc1da38125b982 2024-11-17T02:03:59,072 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1006): Region open journal for 03b7adb3afa9bebd86dc1da38125b982: Running coprocessor pre-open hook at 1731809039057Writing region info on filesystem at 1731809039057Initializing all the Stores at 1731809039059 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731809039059Cleaning up temporary data from old regions at 1731809039066 (+7 ms)Running coprocessor post-open hooks at 1731809039072 (+6 ms)Region opened successfully at 1731809039072 2024-11-17T02:03:59,073 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1731809034900.03b7adb3afa9bebd86dc1da38125b982., pid=74, masterSystemTime=1731809039047 2024-11-17T02:03:59,075 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1731809034900.03b7adb3afa9bebd86dc1da38125b982. 2024-11-17T02:03:59,075 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(153): Opened test,iii,1731809034900.03b7adb3afa9bebd86dc1da38125b982. 2024-11-17T02:03:59,076 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(132): Open test,vvv,1731809034900.2f4fe90ec03839f5d26acd9d02569e56. 2024-11-17T02:03:59,076 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7752): Opening region: {ENCODED => 2f4fe90ec03839f5d26acd9d02569e56, NAME => 'test,vvv,1731809034900.2f4fe90ec03839f5d26acd9d02569e56.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-17T02:03:59,076 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2f4fe90ec03839f5d26acd9d02569e56 2024-11-17T02:03:59,076 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(898): Instantiated test,vvv,1731809034900.2f4fe90ec03839f5d26acd9d02569e56.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:59,076 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7794): checking encryption for 2f4fe90ec03839f5d26acd9d02569e56 2024-11-17T02:03:59,076 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7797): checking classloading for 2f4fe90ec03839f5d26acd9d02569e56 2024-11-17T02:03:59,077 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-11-17T02:03:59,077 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,078 INFO [StoreOpener-2f4fe90ec03839f5d26acd9d02569e56-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2f4fe90ec03839f5d26acd9d02569e56 2024-11-17T02:03:59,077 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,079 INFO [StoreOpener-2f4fe90ec03839f5d26acd9d02569e56-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2f4fe90ec03839f5d26acd9d02569e56 columnFamilyName family 2024-11-17T02:03:59,079 DEBUG [StoreOpener-2f4fe90ec03839f5d26acd9d02569e56-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:59,080 INFO [StoreOpener-2f4fe90ec03839f5d26acd9d02569e56-1 {}] regionserver.HStore(327): Store=2f4fe90ec03839f5d26acd9d02569e56/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T02:03:59,080 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1038): replaying wal for 2f4fe90ec03839f5d26acd9d02569e56 2024-11-17T02:03:59,081 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/2f4fe90ec03839f5d26acd9d02569e56 2024-11-17T02:03:59,082 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/2f4fe90ec03839f5d26acd9d02569e56 2024-11-17T02:03:59,082 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1048): stopping wal replay for 2f4fe90ec03839f5d26acd9d02569e56 2024-11-17T02:03:59,083 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1060): Cleaning up temporary data for 2f4fe90ec03839f5d26acd9d02569e56 2024-11-17T02:03:59,085 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1093): writing seq id for 2f4fe90ec03839f5d26acd9d02569e56 2024-11-17T02:03:59,086 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1114): Opened 2f4fe90ec03839f5d26acd9d02569e56; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64638655, jitterRate=-0.036808982491493225}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T02:03:59,086 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2f4fe90ec03839f5d26acd9d02569e56 2024-11-17T02:03:59,086 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1006): Region open journal for 2f4fe90ec03839f5d26acd9d02569e56: Running coprocessor pre-open hook at 1731809039076Writing region info on filesystem at 1731809039076Initializing all the Stores at 1731809039078 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731809039078Cleaning up temporary data from old regions at 1731809039083 (+5 ms)Running coprocessor post-open hooks at 1731809039086 (+3 ms)Region opened successfully at 1731809039086 2024-11-17T02:03:59,087 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1731809034900.2f4fe90ec03839f5d26acd9d02569e56., pid=72, masterSystemTime=1731809039047 2024-11-17T02:03:59,089 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1731809034900.2f4fe90ec03839f5d26acd9d02569e56. 2024-11-17T02:03:59,089 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(153): Opened test,vvv,1731809034900.2f4fe90ec03839f5d26acd9d02569e56. 2024-11-17T02:03:59,089 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,ooo,1731809034900.25a67f85380b4a3cd6890ab81d0b3a27. 2024-11-17T02:03:59,089 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => 25a67f85380b4a3cd6890ab81d0b3a27, NAME => 'test,ooo,1731809034900.25a67f85380b4a3cd6890ab81d0b3a27.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-17T02:03:59,090 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 25a67f85380b4a3cd6890ab81d0b3a27 2024-11-17T02:03:59,088 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 4 more 2024-11-17T02:03:59,090 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,ooo,1731809034900.25a67f85380b4a3cd6890ab81d0b3a27.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:59,090 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for 25a67f85380b4a3cd6890ab81d0b3a27 2024-11-17T02:03:59,090 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for 25a67f85380b4a3cd6890ab81d0b3a27 2024-11-17T02:03:59,090 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-17T02:03:59,090 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure b6eba00b85aa16d5ccbfc8e3c35ff298, server=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,090 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,091 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure b6eba00b85aa16d5ccbfc8e3c35ff298, server=c6c0d7a0a7c0,45037,1731809032394 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,092 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,fff,1731809034900.112f5bbe15f72793585db9ba98950976. 2024-11-17T02:03:59,092 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => 112f5bbe15f72793585db9ba98950976, NAME => 'test,fff,1731809034900.112f5bbe15f72793585db9ba98950976.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-17T02:03:59,092 INFO [StoreOpener-25a67f85380b4a3cd6890ab81d0b3a27-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 25a67f85380b4a3cd6890ab81d0b3a27 2024-11-17T02:03:59,092 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 112f5bbe15f72793585db9ba98950976 2024-11-17T02:03:59,093 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,fff,1731809034900.112f5bbe15f72793585db9ba98950976.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:59,093 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for 112f5bbe15f72793585db9ba98950976 2024-11-17T02:03:59,093 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for 112f5bbe15f72793585db9ba98950976 2024-11-17T02:03:59,094 INFO [StoreOpener-25a67f85380b4a3cd6890ab81d0b3a27-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25a67f85380b4a3cd6890ab81d0b3a27 columnFamilyName family 2024-11-17T02:03:59,094 DEBUG [StoreOpener-25a67f85380b4a3cd6890ab81d0b3a27-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:59,094 INFO [StoreOpener-112f5bbe15f72793585db9ba98950976-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 112f5bbe15f72793585db9ba98950976 2024-11-17T02:03:59,094 INFO [StoreOpener-25a67f85380b4a3cd6890ab81d0b3a27-1 {}] regionserver.HStore(327): Store=25a67f85380b4a3cd6890ab81d0b3a27/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T02:03:59,094 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for 25a67f85380b4a3cd6890ab81d0b3a27 2024-11-17T02:03:59,095 INFO [StoreOpener-112f5bbe15f72793585db9ba98950976-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 112f5bbe15f72793585db9ba98950976 columnFamilyName family 2024-11-17T02:03:59,095 DEBUG [StoreOpener-112f5bbe15f72793585db9ba98950976-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:59,095 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/25a67f85380b4a3cd6890ab81d0b3a27 2024-11-17T02:03:59,096 INFO [StoreOpener-112f5bbe15f72793585db9ba98950976-1 {}] regionserver.HStore(327): Store=112f5bbe15f72793585db9ba98950976/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T02:03:59,096 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for 112f5bbe15f72793585db9ba98950976 2024-11-17T02:03:59,098 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/112f5bbe15f72793585db9ba98950976 2024-11-17T02:03:59,098 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/25a67f85380b4a3cd6890ab81d0b3a27 2024-11-17T02:03:59,098 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for 25a67f85380b4a3cd6890ab81d0b3a27 2024-11-17T02:03:59,098 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for 25a67f85380b4a3cd6890ab81d0b3a27 2024-11-17T02:03:59,099 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/112f5bbe15f72793585db9ba98950976 2024-11-17T02:03:59,099 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for 112f5bbe15f72793585db9ba98950976 2024-11-17T02:03:59,099 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for 112f5bbe15f72793585db9ba98950976 2024-11-17T02:03:59,100 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for 25a67f85380b4a3cd6890ab81d0b3a27 2024-11-17T02:03:59,101 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened 25a67f85380b4a3cd6890ab81d0b3a27; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67479284, jitterRate=0.005519688129425049}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T02:03:59,101 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 25a67f85380b4a3cd6890ab81d0b3a27 2024-11-17T02:03:59,101 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for 112f5bbe15f72793585db9ba98950976 2024-11-17T02:03:59,101 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for 25a67f85380b4a3cd6890ab81d0b3a27: Running coprocessor pre-open hook at 1731809039090Writing region info on filesystem at 1731809039090Initializing all the Stores at 1731809039092 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731809039092Cleaning up temporary data from old regions at 1731809039098 (+6 ms)Running coprocessor post-open hooks at 1731809039101 (+3 ms)Region opened successfully at 1731809039101 2024-11-17T02:03:59,102 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1731809034900.25a67f85380b4a3cd6890ab81d0b3a27., pid=75, masterSystemTime=1731809039047 2024-11-17T02:03:59,102 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened 112f5bbe15f72793585db9ba98950976; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73371486, jitterRate=0.0933203399181366}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T02:03:59,102 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 112f5bbe15f72793585db9ba98950976 2024-11-17T02:03:59,102 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for 112f5bbe15f72793585db9ba98950976: Running coprocessor pre-open hook at 1731809039093Writing region info on filesystem at 1731809039093Initializing all the Stores at 1731809039094 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731809039094Cleaning up temporary data from old regions at 1731809039099 (+5 ms)Running coprocessor post-open hooks at 1731809039102 (+3 ms)Region opened successfully at 1731809039102 2024-11-17T02:03:59,103 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1731809034900.112f5bbe15f72793585db9ba98950976., pid=77, masterSystemTime=1731809039087 2024-11-17T02:03:59,104 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1731809034900.25a67f85380b4a3cd6890ab81d0b3a27. 2024-11-17T02:03:59,104 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,ooo,1731809034900.25a67f85380b4a3cd6890ab81d0b3a27. 2024-11-17T02:03:59,104 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(132): Open test,,1731809034900.de9924be32211cddceef43e5b4866dc8. 2024-11-17T02:03:59,104 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7752): Opening region: {ENCODED => de9924be32211cddceef43e5b4866dc8, NAME => 'test,,1731809034900.de9924be32211cddceef43e5b4866dc8.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-17T02:03:59,105 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test de9924be32211cddceef43e5b4866dc8 2024-11-17T02:03:59,105 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(898): Instantiated test,,1731809034900.de9924be32211cddceef43e5b4866dc8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:59,105 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7794): checking encryption for de9924be32211cddceef43e5b4866dc8 2024-11-17T02:03:59,105 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7797): checking classloading for de9924be32211cddceef43e5b4866dc8 2024-11-17T02:03:59,106 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1731809034900.112f5bbe15f72793585db9ba98950976. 2024-11-17T02:03:59,106 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened test,fff,1731809034900.112f5bbe15f72793585db9ba98950976. 2024-11-17T02:03:59,106 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,sss,1731809034900.b6eba00b85aa16d5ccbfc8e3c35ff298. 2024-11-17T02:03:59,106 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => b6eba00b85aa16d5ccbfc8e3c35ff298, NAME => 'test,sss,1731809034900.b6eba00b85aa16d5ccbfc8e3c35ff298.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-17T02:03:59,107 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b6eba00b85aa16d5ccbfc8e3c35ff298 2024-11-17T02:03:59,107 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,sss,1731809034900.b6eba00b85aa16d5ccbfc8e3c35ff298.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:59,107 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for b6eba00b85aa16d5ccbfc8e3c35ff298 2024-11-17T02:03:59,107 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for b6eba00b85aa16d5ccbfc8e3c35ff298 2024-11-17T02:03:59,107 INFO [StoreOpener-de9924be32211cddceef43e5b4866dc8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region de9924be32211cddceef43e5b4866dc8 2024-11-17T02:03:59,108 INFO [StoreOpener-de9924be32211cddceef43e5b4866dc8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de9924be32211cddceef43e5b4866dc8 columnFamilyName family 2024-11-17T02:03:59,108 DEBUG [StoreOpener-de9924be32211cddceef43e5b4866dc8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:59,108 INFO [StoreOpener-b6eba00b85aa16d5ccbfc8e3c35ff298-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b6eba00b85aa16d5ccbfc8e3c35ff298 2024-11-17T02:03:59,109 INFO [StoreOpener-de9924be32211cddceef43e5b4866dc8-1 {}] regionserver.HStore(327): Store=de9924be32211cddceef43e5b4866dc8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T02:03:59,109 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1038): replaying wal for de9924be32211cddceef43e5b4866dc8 2024-11-17T02:03:59,109 INFO [StoreOpener-b6eba00b85aa16d5ccbfc8e3c35ff298-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b6eba00b85aa16d5ccbfc8e3c35ff298 columnFamilyName family 2024-11-17T02:03:59,109 DEBUG [StoreOpener-b6eba00b85aa16d5ccbfc8e3c35ff298-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:59,110 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/de9924be32211cddceef43e5b4866dc8 2024-11-17T02:03:59,110 INFO [StoreOpener-b6eba00b85aa16d5ccbfc8e3c35ff298-1 {}] regionserver.HStore(327): Store=b6eba00b85aa16d5ccbfc8e3c35ff298/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T02:03:59,110 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for b6eba00b85aa16d5ccbfc8e3c35ff298 2024-11-17T02:03:59,111 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/de9924be32211cddceef43e5b4866dc8 2024-11-17T02:03:59,111 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/b6eba00b85aa16d5ccbfc8e3c35ff298 2024-11-17T02:03:59,111 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1048): stopping wal replay for de9924be32211cddceef43e5b4866dc8 2024-11-17T02:03:59,111 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1060): Cleaning up temporary data for de9924be32211cddceef43e5b4866dc8 2024-11-17T02:03:59,112 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/b6eba00b85aa16d5ccbfc8e3c35ff298 2024-11-17T02:03:59,112 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for b6eba00b85aa16d5ccbfc8e3c35ff298 2024-11-17T02:03:59,112 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for b6eba00b85aa16d5ccbfc8e3c35ff298 2024-11-17T02:03:59,114 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1093): writing seq id for de9924be32211cddceef43e5b4866dc8 2024-11-17T02:03:59,114 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for b6eba00b85aa16d5ccbfc8e3c35ff298 2024-11-17T02:03:59,114 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1114): Opened de9924be32211cddceef43e5b4866dc8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67689401, jitterRate=0.008650675415992737}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T02:03:59,115 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1122): Running coprocessor post-open hooks for de9924be32211cddceef43e5b4866dc8 2024-11-17T02:03:59,115 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1006): Region open journal for de9924be32211cddceef43e5b4866dc8: Running coprocessor pre-open hook at 1731809039105Writing region info on filesystem at 1731809039105Initializing all the Stores at 1731809039107 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731809039107Cleaning up temporary data from old regions at 1731809039111 (+4 ms)Running coprocessor post-open hooks at 1731809039115 (+4 ms)Region opened successfully at 1731809039115 2024-11-17T02:03:59,115 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened b6eba00b85aa16d5ccbfc8e3c35ff298; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59300927, jitterRate=-0.11634732782840729}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T02:03:59,115 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b6eba00b85aa16d5ccbfc8e3c35ff298 2024-11-17T02:03:59,115 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for b6eba00b85aa16d5ccbfc8e3c35ff298: Running coprocessor pre-open hook at 1731809039107Writing region info on filesystem at 1731809039107Initializing all the Stores at 1731809039108 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731809039108Cleaning up temporary data from old regions at 1731809039112 (+4 ms)Running coprocessor post-open hooks at 1731809039115 (+3 ms)Region opened successfully at 1731809039115 2024-11-17T02:03:59,116 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1731809034900.de9924be32211cddceef43e5b4866dc8., pid=73, masterSystemTime=1731809039047 2024-11-17T02:03:59,116 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1731809034900.b6eba00b85aa16d5ccbfc8e3c35ff298., pid=79, masterSystemTime=1731809039087 2024-11-17T02:03:59,118 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1731809034900.de9924be32211cddceef43e5b4866dc8. 2024-11-17T02:03:59,118 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(153): Opened test,,1731809034900.de9924be32211cddceef43e5b4866dc8. 2024-11-17T02:03:59,119 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1731809034900.b6eba00b85aa16d5ccbfc8e3c35ff298. 2024-11-17T02:03:59,119 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened test,sss,1731809034900.b6eba00b85aa16d5ccbfc8e3c35ff298. 2024-11-17T02:03:59,119 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,yyy,1731809034900.7eed07a820ab15d7af64462e2c1b70f7. 2024-11-17T02:03:59,119 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => 7eed07a820ab15d7af64462e2c1b70f7, NAME => 'test,yyy,1731809034900.7eed07a820ab15d7af64462e2c1b70f7.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-17T02:03:59,120 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7eed07a820ab15d7af64462e2c1b70f7 2024-11-17T02:03:59,120 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,yyy,1731809034900.7eed07a820ab15d7af64462e2c1b70f7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:59,120 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for 7eed07a820ab15d7af64462e2c1b70f7 2024-11-17T02:03:59,120 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for 7eed07a820ab15d7af64462e2c1b70f7 2024-11-17T02:03:59,121 INFO [StoreOpener-7eed07a820ab15d7af64462e2c1b70f7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7eed07a820ab15d7af64462e2c1b70f7 2024-11-17T02:03:59,123 INFO [StoreOpener-7eed07a820ab15d7af64462e2c1b70f7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7eed07a820ab15d7af64462e2c1b70f7 columnFamilyName family 2024-11-17T02:03:59,123 DEBUG [StoreOpener-7eed07a820ab15d7af64462e2c1b70f7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:59,123 INFO [StoreOpener-7eed07a820ab15d7af64462e2c1b70f7-1 {}] regionserver.HStore(327): Store=7eed07a820ab15d7af64462e2c1b70f7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T02:03:59,124 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for 7eed07a820ab15d7af64462e2c1b70f7 2024-11-17T02:03:59,124 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/7eed07a820ab15d7af64462e2c1b70f7 2024-11-17T02:03:59,126 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/7eed07a820ab15d7af64462e2c1b70f7 2024-11-17T02:03:59,126 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for 7eed07a820ab15d7af64462e2c1b70f7 2024-11-17T02:03:59,126 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for 7eed07a820ab15d7af64462e2c1b70f7 2024-11-17T02:03:59,128 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for 7eed07a820ab15d7af64462e2c1b70f7 2024-11-17T02:03:59,129 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened 7eed07a820ab15d7af64462e2c1b70f7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61108860, jitterRate=-0.08940702676773071}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T02:03:59,129 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7eed07a820ab15d7af64462e2c1b70f7 2024-11-17T02:03:59,129 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for 7eed07a820ab15d7af64462e2c1b70f7: Running coprocessor pre-open hook at 1731809039120Writing region info on filesystem at 1731809039120Initializing all the Stores at 1731809039121 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731809039121Cleaning up temporary data from old regions at 1731809039126 (+5 ms)Running coprocessor post-open hooks at 1731809039129 (+3 ms)Region opened successfully at 1731809039129 2024-11-17T02:03:59,130 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1731809034900.7eed07a820ab15d7af64462e2c1b70f7., pid=76, masterSystemTime=1731809039087 2024-11-17T02:03:59,132 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1731809034900.7eed07a820ab15d7af64462e2c1b70f7. 2024-11-17T02:03:59,132 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened test,yyy,1731809034900.7eed07a820ab15d7af64462e2c1b70f7. 2024-11-17T02:03:59,132 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,lll,1731809034900.1056118c93716b24d18f749826cf316a. 2024-11-17T02:03:59,132 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => 1056118c93716b24d18f749826cf316a, NAME => 'test,lll,1731809034900.1056118c93716b24d18f749826cf316a.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-17T02:03:59,133 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1056118c93716b24d18f749826cf316a 2024-11-17T02:03:59,133 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,lll,1731809034900.1056118c93716b24d18f749826cf316a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T02:03:59,133 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for 1056118c93716b24d18f749826cf316a 2024-11-17T02:03:59,133 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for 1056118c93716b24d18f749826cf316a 2024-11-17T02:03:59,135 INFO [StoreOpener-1056118c93716b24d18f749826cf316a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1056118c93716b24d18f749826cf316a 2024-11-17T02:03:59,136 INFO [StoreOpener-1056118c93716b24d18f749826cf316a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1056118c93716b24d18f749826cf316a columnFamilyName family 2024-11-17T02:03:59,136 DEBUG [StoreOpener-1056118c93716b24d18f749826cf316a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T02:03:59,136 INFO [StoreOpener-1056118c93716b24d18f749826cf316a-1 {}] regionserver.HStore(327): Store=1056118c93716b24d18f749826cf316a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T02:03:59,136 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for 1056118c93716b24d18f749826cf316a 2024-11-17T02:03:59,137 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/1056118c93716b24d18f749826cf316a 2024-11-17T02:03:59,139 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/1056118c93716b24d18f749826cf316a 2024-11-17T02:03:59,139 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for 1056118c93716b24d18f749826cf316a 2024-11-17T02:03:59,139 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for 1056118c93716b24d18f749826cf316a 2024-11-17T02:03:59,142 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for 1056118c93716b24d18f749826cf316a 2024-11-17T02:03:59,143 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened 1056118c93716b24d18f749826cf316a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62127218, jitterRate=-0.0742323100566864}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T02:03:59,143 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1056118c93716b24d18f749826cf316a 2024-11-17T02:03:59,143 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for 1056118c93716b24d18f749826cf316a: Running coprocessor pre-open hook at 1731809039133Writing region info on filesystem at 1731809039133Initializing all the Stores at 1731809039134 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731809039134Cleaning up temporary data from old regions at 1731809039139 (+5 ms)Running coprocessor post-open hooks at 1731809039143 (+4 ms)Region opened successfully at 1731809039143 2024-11-17T02:03:59,144 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1731809034900.1056118c93716b24d18f749826cf316a., pid=78, masterSystemTime=1731809039087 2024-11-17T02:03:59,147 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1731809034900.1056118c93716b24d18f749826cf316a. 2024-11-17T02:03:59,147 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened test,lll,1731809034900.1056118c93716b24d18f749826cf316a. 2024-11-17T02:03:59,187 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-17T02:03:59,187 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,187 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-17T02:03:59,187 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-17T02:03:59,188 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-17T02:03:59,188 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-17T02:03:59,188 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-17T02:03:59,188 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-17T02:03:59,189 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-17T02:03:59,196 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-17T02:03:59,197 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,197 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,197 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,197 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,197 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,197 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,197 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,197 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,197 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,197 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,197 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,197 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,198 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,198 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,198 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,198 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,198 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-17T02:03:59,198 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-17T02:03:59,198 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,198 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,198 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-17T02:03:59,198 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-17T02:03:59,199 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-17T02:03:59,201 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-17T02:03:59,206 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true 2024-11-17T02:03:59,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T02:03:59,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN 2024-11-17T02:03:59,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN 2024-11-17T02:03:59,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN 2024-11-17T02:03:59,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN 2024-11-17T02:03:59,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN 2024-11-17T02:03:59,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN 2024-11-17T02:03:59,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN 2024-11-17T02:03:59,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN 2024-11-17T02:03:59,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2f4fe90ec03839f5d26acd9d02569e56, server=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure de9924be32211cddceef43e5b4866dc8, server=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 25a67f85380b4a3cd6890ab81d0b3a27, server=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7eed07a820ab15d7af64462e2c1b70f7, server=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 112f5bbe15f72793585db9ba98950976, server=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1056118c93716b24d18f749826cf316a, server=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,208 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure b6eba00b85aa16d5ccbfc8e3c35ff298, server=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,208 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,208 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,208 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,208 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,208 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,208 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,208 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,208 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,208 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN 2024-11-17T02:03:59,208 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,208 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN 2024-11-17T02:03:59,208 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,208 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN 2024-11-17T02:03:59,208 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,208 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN 2024-11-17T02:03:59,208 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,209 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN 2024-11-17T02:03:59,209 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,209 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN 2024-11-17T02:03:59,209 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,209 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN 2024-11-17T02:03:59,209 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,209 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN 2024-11-17T02:03:59,209 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2f4fe90ec03839f5d26acd9d02569e56, server=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,209 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure de9924be32211cddceef43e5b4866dc8, server=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,209 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,209 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 25a67f85380b4a3cd6890ab81d0b3a27, server=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,209 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7eed07a820ab15d7af64462e2c1b70f7, server=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,209 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 112f5bbe15f72793585db9ba98950976, server=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,209 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1056118c93716b24d18f749826cf316a, server=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,209 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure b6eba00b85aa16d5ccbfc8e3c35ff298, server=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,210 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 20 msec 2024-11-17T02:03:59,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=61 2024-11-17T02:03:59,213 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c6c0d7a0a7c0,45037,1731809032394, table=hbase:meta, region=1588230740 2024-11-17T02:03:59,213 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c6c0d7a0a7c0,45037,1731809032394, state=OPEN 2024-11-17T02:03:59,236 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,236 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,236 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,236 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,236 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,236 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,237 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN to state=OFFLINE, location=null, table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298 to restore RIT 2024-11-17T02:03:59,238 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN to state=OFFLINE, location=null, table=test, region=03b7adb3afa9bebd86dc1da38125b982 to restore RIT 2024-11-17T02:03:59,238 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN to state=OFFLINE, location=null, table=test, region=2f4fe90ec03839f5d26acd9d02569e56 to restore RIT 2024-11-17T02:03:59,238 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN to state=OFFLINE, location=null, table=test, region=25a67f85380b4a3cd6890ab81d0b3a27 to restore RIT 2024-11-17T02:03:59,238 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN to state=OFFLINE, location=null, table=test, region=de9924be32211cddceef43e5b4866dc8 to restore RIT 2024-11-17T02:03:59,238 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN to state=OFFLINE, location=null, table=test, region=1056118c93716b24d18f749826cf316a to restore RIT 2024-11-17T02:03:59,238 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN to state=OFFLINE, location=null, table=test, region=7eed07a820ab15d7af64462e2c1b70f7 to restore RIT 2024-11-17T02:03:59,238 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN to state=OFFLINE, location=null, table=test, region=112f5bbe15f72793585db9ba98950976 to restore RIT 2024-11-17T02:03:59,239 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731809069239 2024-11-17T02:03:59,239 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-17T02:03:59,239 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,241 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-17T02:03:59,242 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-17T02:03:59,242 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1056118c93716b24d18f749826cf316a, server=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,242 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1056118c93716b24d18f749826cf316a, server=c6c0d7a0a7c0,45037,1731809032394 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,263 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9924be32211cddceef43e5b4866dc8, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,263 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a17ac9e270344fa2752f51f8196693b2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,264 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81f322eb57dcb42c3b4269a738bafd6d, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,264 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15c64fab21a3ca03ec0a7afabc19cdc9, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,265 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8356c0fdb856c376b8678f2c2597565a, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,265 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=112f5bbe15f72793585db9ba98950976, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,265 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7517bce9e04879764302cccd45af20fb, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,266 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fd33da09f9d938215ac0e7127ad70d5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,266 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03b7adb3afa9bebd86dc1da38125b982, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,266 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05534a4a027a1fdc90227b7a2e681933, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,267 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac10d67ff6c3f527cc0861e738032bdb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,267 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1056118c93716b24d18f749826cf316a, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,267 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663bac36df6c9873ad776bb3ad42409f, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,267 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74108f0b014075d4993f0eee396e39e5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,268 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=25a67f85380b4a3cd6890ab81d0b3a27, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,268 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e1c1fc49350743a9cde50c433c8cb19, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,268 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=899f31eacba6d2d0aedbee7d8bbb4c9d, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,269 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7934444caa228b9e5d6d7b28b59a796, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,269 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6eba00b85aa16d5ccbfc8e3c35ff298, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,269 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4748cc4c5ad3838a3d0568cee797c169, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,269 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77bd97fde6bd169f3e20553b45a65e2c, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,270 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f4fe90ec03839f5d26acd9d02569e56, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,270 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db157dde5b6770afbf869f7733d95efb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,270 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d86e1fe125f78846dcaa5b9921c72db, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,270 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eed07a820ab15d7af64462e2c1b70f7, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,271 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=037b7a93730957425bc54697b3594bb2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,271 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-17T02:03:59,271 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731809099271 2024-11-17T02:03:59,271 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731809159271 2024-11-17T02:03:59,271 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 32 msec 2024-11-17T02:03:59,271 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-17T02:03:59,271 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,271 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-17T02:03:59,271 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-17T02:03:59,272 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-17T02:03:59,272 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-17T02:03:59,272 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-17T02:03:59,272 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-17T02:03:59,273 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-17T02:03:59,279 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-17T02:03:59,279 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,279 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,279 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,279 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,279 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,279 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,279 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,279 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,279 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,279 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,279 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,279 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,279 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,280 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,280 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,280 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-17T02:03:59,280 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-17T02:03:59,280 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,280 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,280 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-17T02:03:59,280 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-17T02:03:59,280 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-17T02:03:59,282 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-17T02:03:59,284 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true 2024-11-17T02:03:59,285 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,285 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,285 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T02:03:59,285 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,285 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,285 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,285 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN 2024-11-17T02:03:59,285 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN 2024-11-17T02:03:59,285 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN 2024-11-17T02:03:59,285 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN 2024-11-17T02:03:59,285 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN 2024-11-17T02:03:59,285 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN 2024-11-17T02:03:59,285 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN 2024-11-17T02:03:59,285 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN 2024-11-17T02:03:59,285 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2f4fe90ec03839f5d26acd9d02569e56, server=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,285 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure de9924be32211cddceef43e5b4866dc8, server=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,285 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,285 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 25a67f85380b4a3cd6890ab81d0b3a27, server=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,285 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7eed07a820ab15d7af64462e2c1b70f7, server=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,285 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 112f5bbe15f72793585db9ba98950976, server=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,285 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1056118c93716b24d18f749826cf316a, server=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,285 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,285 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,285 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,285 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,285 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,285 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,286 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,286 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,286 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN 2024-11-17T02:03:59,286 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,286 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN 2024-11-17T02:03:59,286 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,286 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN 2024-11-17T02:03:59,286 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,286 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN 2024-11-17T02:03:59,286 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,286 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN 2024-11-17T02:03:59,286 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,286 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN 2024-11-17T02:03:59,286 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,286 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN 2024-11-17T02:03:59,286 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,286 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN 2024-11-17T02:03:59,286 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2f4fe90ec03839f5d26acd9d02569e56, server=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,286 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure de9924be32211cddceef43e5b4866dc8, server=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,286 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,286 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 25a67f85380b4a3cd6890ab81d0b3a27, server=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,286 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7eed07a820ab15d7af64462e2c1b70f7, server=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,286 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 112f5bbe15f72793585db9ba98950976, server=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,286 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1056118c93716b24d18f749826cf316a, server=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,286 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 13 msec 2024-11-17T02:03:59,287 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=61 2024-11-17T02:03:59,287 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c6c0d7a0a7c0,45037,1731809032394, table=hbase:meta, region=1588230740 2024-11-17T02:03:59,288 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c6c0d7a0a7c0,45037,1731809032394, state=OPEN 2024-11-17T02:03:59,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,310 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,310 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,310 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,311 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN to state=OFFLINE, location=null, table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298 to restore RIT 2024-11-17T02:03:59,311 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN to state=OFFLINE, location=null, table=test, region=03b7adb3afa9bebd86dc1da38125b982 to restore RIT 2024-11-17T02:03:59,311 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN to state=OFFLINE, location=null, table=test, region=2f4fe90ec03839f5d26acd9d02569e56 to restore RIT 2024-11-17T02:03:59,311 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN to state=OFFLINE, location=null, table=test, region=25a67f85380b4a3cd6890ab81d0b3a27 to restore RIT 2024-11-17T02:03:59,311 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN to state=OFFLINE, location=null, table=test, region=de9924be32211cddceef43e5b4866dc8 to restore RIT 2024-11-17T02:03:59,311 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN to state=OFFLINE, location=null, table=test, region=1056118c93716b24d18f749826cf316a to restore RIT 2024-11-17T02:03:59,311 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN to state=OFFLINE, location=null, table=test, region=7eed07a820ab15d7af64462e2c1b70f7 to restore RIT 2024-11-17T02:03:59,311 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN to state=OFFLINE, location=null, table=test, region=112f5bbe15f72793585db9ba98950976 to restore RIT 2024-11-17T02:03:59,311 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731809069311 2024-11-17T02:03:59,311 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-17T02:03:59,311 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,313 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-17T02:03:59,314 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-17T02:03:59,314 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 112f5bbe15f72793585db9ba98950976, server=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,314 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,315 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 112f5bbe15f72793585db9ba98950976, server=c6c0d7a0a7c0,45037,1731809032394 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9924be32211cddceef43e5b4866dc8, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a17ac9e270344fa2752f51f8196693b2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81f322eb57dcb42c3b4269a738bafd6d, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15c64fab21a3ca03ec0a7afabc19cdc9, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8356c0fdb856c376b8678f2c2597565a, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=112f5bbe15f72793585db9ba98950976, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7517bce9e04879764302cccd45af20fb, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fd33da09f9d938215ac0e7127ad70d5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03b7adb3afa9bebd86dc1da38125b982, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05534a4a027a1fdc90227b7a2e681933, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac10d67ff6c3f527cc0861e738032bdb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1056118c93716b24d18f749826cf316a, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663bac36df6c9873ad776bb3ad42409f, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74108f0b014075d4993f0eee396e39e5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=25a67f85380b4a3cd6890ab81d0b3a27, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e1c1fc49350743a9cde50c433c8cb19, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,323 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=899f31eacba6d2d0aedbee7d8bbb4c9d, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,323 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7934444caa228b9e5d6d7b28b59a796, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,323 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6eba00b85aa16d5ccbfc8e3c35ff298, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,323 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4748cc4c5ad3838a3d0568cee797c169, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,323 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77bd97fde6bd169f3e20553b45a65e2c, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,323 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f4fe90ec03839f5d26acd9d02569e56, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,324 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db157dde5b6770afbf869f7733d95efb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,324 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d86e1fe125f78846dcaa5b9921c72db, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,324 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eed07a820ab15d7af64462e2c1b70f7, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,324 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=037b7a93730957425bc54697b3594bb2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,324 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-17T02:03:59,324 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731809099324 2024-11-17T02:03:59,324 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731809159324 2024-11-17T02:03:59,324 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-11-17T02:03:59,324 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-17T02:03:59,324 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,324 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-17T02:03:59,324 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-17T02:03:59,325 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-17T02:03:59,325 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-17T02:03:59,325 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-17T02:03:59,325 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-17T02:03:59,326 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-17T02:03:59,329 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-17T02:03:59,329 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,329 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,329 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,329 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,329 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,329 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,330 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,330 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,330 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,330 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,330 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,330 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,330 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,330 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,330 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-17T02:03:59,330 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-17T02:03:59,330 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,330 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,330 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-17T02:03:59,330 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-17T02:03:59,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-17T02:03:59,331 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-17T02:03:59,333 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true 2024-11-17T02:03:59,333 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T02:03:59,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN 2024-11-17T02:03:59,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN 2024-11-17T02:03:59,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN 2024-11-17T02:03:59,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN 2024-11-17T02:03:59,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN 2024-11-17T02:03:59,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN 2024-11-17T02:03:59,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN 2024-11-17T02:03:59,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN 2024-11-17T02:03:59,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2f4fe90ec03839f5d26acd9d02569e56, server=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure de9924be32211cddceef43e5b4866dc8, server=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 25a67f85380b4a3cd6890ab81d0b3a27, server=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7eed07a820ab15d7af64462e2c1b70f7, server=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 112f5bbe15f72793585db9ba98950976, server=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,334 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,334 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,334 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,334 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,335 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,335 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,335 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,335 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,335 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN 2024-11-17T02:03:59,335 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,335 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN 2024-11-17T02:03:59,335 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,335 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN 2024-11-17T02:03:59,335 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,335 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN 2024-11-17T02:03:59,335 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,335 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN 2024-11-17T02:03:59,335 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,335 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN 2024-11-17T02:03:59,335 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,335 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN 2024-11-17T02:03:59,335 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,335 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN 2024-11-17T02:03:59,335 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2f4fe90ec03839f5d26acd9d02569e56, server=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,335 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure de9924be32211cddceef43e5b4866dc8, server=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,335 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,335 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 25a67f85380b4a3cd6890ab81d0b3a27, server=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,335 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7eed07a820ab15d7af64462e2c1b70f7, server=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,335 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 112f5bbe15f72793585db9ba98950976, server=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,335 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-17T02:03:59,336 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=61 2024-11-17T02:03:59,336 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c6c0d7a0a7c0,45037,1731809032394, table=hbase:meta, region=1588230740 2024-11-17T02:03:59,337 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c6c0d7a0a7c0,45037,1731809032394, state=OPEN 2024-11-17T02:03:59,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,344 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,344 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,344 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,345 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN to state=OFFLINE, location=null, table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298 to restore RIT 2024-11-17T02:03:59,345 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN to state=OFFLINE, location=null, table=test, region=03b7adb3afa9bebd86dc1da38125b982 to restore RIT 2024-11-17T02:03:59,345 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN to state=OFFLINE, location=null, table=test, region=2f4fe90ec03839f5d26acd9d02569e56 to restore RIT 2024-11-17T02:03:59,345 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN to state=OFFLINE, location=null, table=test, region=25a67f85380b4a3cd6890ab81d0b3a27 to restore RIT 2024-11-17T02:03:59,345 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN to state=OFFLINE, location=null, table=test, region=de9924be32211cddceef43e5b4866dc8 to restore RIT 2024-11-17T02:03:59,345 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN to state=OFFLINE, location=null, table=test, region=1056118c93716b24d18f749826cf316a to restore RIT 2024-11-17T02:03:59,345 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN to state=OFFLINE, location=null, table=test, region=7eed07a820ab15d7af64462e2c1b70f7 to restore RIT 2024-11-17T02:03:59,345 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN to state=OFFLINE, location=null, table=test, region=112f5bbe15f72793585db9ba98950976 to restore RIT 2024-11-17T02:03:59,345 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731809069345 2024-11-17T02:03:59,345 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-17T02:03:59,345 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,347 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-17T02:03:59,348 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-17T02:03:59,348 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7eed07a820ab15d7af64462e2c1b70f7, server=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,348 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7eed07a820ab15d7af64462e2c1b70f7, server=c6c0d7a0a7c0,45037,1731809032394 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,354 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9924be32211cddceef43e5b4866dc8, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,354 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a17ac9e270344fa2752f51f8196693b2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,355 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81f322eb57dcb42c3b4269a738bafd6d, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,355 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15c64fab21a3ca03ec0a7afabc19cdc9, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,355 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8356c0fdb856c376b8678f2c2597565a, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,355 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=112f5bbe15f72793585db9ba98950976, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,355 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7517bce9e04879764302cccd45af20fb, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,355 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fd33da09f9d938215ac0e7127ad70d5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,356 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03b7adb3afa9bebd86dc1da38125b982, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,356 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05534a4a027a1fdc90227b7a2e681933, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,356 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac10d67ff6c3f527cc0861e738032bdb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,356 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1056118c93716b24d18f749826cf316a, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,357 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663bac36df6c9873ad776bb3ad42409f, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,357 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74108f0b014075d4993f0eee396e39e5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,357 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=25a67f85380b4a3cd6890ab81d0b3a27, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,357 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e1c1fc49350743a9cde50c433c8cb19, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,358 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=899f31eacba6d2d0aedbee7d8bbb4c9d, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,358 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7934444caa228b9e5d6d7b28b59a796, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,358 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6eba00b85aa16d5ccbfc8e3c35ff298, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,358 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4748cc4c5ad3838a3d0568cee797c169, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,358 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77bd97fde6bd169f3e20553b45a65e2c, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,359 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f4fe90ec03839f5d26acd9d02569e56, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,359 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db157dde5b6770afbf869f7733d95efb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,359 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d86e1fe125f78846dcaa5b9921c72db, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,359 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eed07a820ab15d7af64462e2c1b70f7, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,360 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=037b7a93730957425bc54697b3594bb2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,360 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-17T02:03:59,360 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731809099360 2024-11-17T02:03:59,360 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731809159360 2024-11-17T02:03:59,360 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 14 msec 2024-11-17T02:03:59,360 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-17T02:03:59,360 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,360 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-17T02:03:59,360 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-17T02:03:59,360 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-17T02:03:59,361 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-17T02:03:59,361 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-17T02:03:59,361 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-17T02:03:59,362 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-17T02:03:59,365 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-17T02:03:59,365 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,365 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,366 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,366 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,366 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,366 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,366 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,366 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,366 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,366 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,366 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,366 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,366 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,366 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-17T02:03:59,366 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-17T02:03:59,366 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,366 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,366 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-17T02:03:59,366 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-17T02:03:59,366 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-17T02:03:59,368 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-17T02:03:59,371 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true 2024-11-17T02:03:59,371 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,371 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,371 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T02:03:59,371 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,371 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,371 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,371 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN 2024-11-17T02:03:59,371 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN 2024-11-17T02:03:59,371 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN 2024-11-17T02:03:59,371 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN 2024-11-17T02:03:59,371 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN 2024-11-17T02:03:59,371 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN 2024-11-17T02:03:59,372 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN 2024-11-17T02:03:59,372 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN 2024-11-17T02:03:59,372 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2f4fe90ec03839f5d26acd9d02569e56, server=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,372 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure de9924be32211cddceef43e5b4866dc8, server=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,372 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,372 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 25a67f85380b4a3cd6890ab81d0b3a27, server=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,372 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7eed07a820ab15d7af64462e2c1b70f7, server=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,372 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,372 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,372 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,372 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,372 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,372 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,372 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,372 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,372 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN 2024-11-17T02:03:59,372 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,372 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN 2024-11-17T02:03:59,372 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,372 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN 2024-11-17T02:03:59,372 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,372 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN 2024-11-17T02:03:59,373 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,373 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN 2024-11-17T02:03:59,373 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,373 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN 2024-11-17T02:03:59,373 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,373 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN 2024-11-17T02:03:59,373 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,373 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN 2024-11-17T02:03:59,373 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2f4fe90ec03839f5d26acd9d02569e56, server=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,373 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure de9924be32211cddceef43e5b4866dc8, server=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,373 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,373 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 25a67f85380b4a3cd6890ab81d0b3a27, server=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,373 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7eed07a820ab15d7af64462e2c1b70f7, server=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,373 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32803 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,373 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-11-17T02:03:59,374 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32803 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,375 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=61 2024-11-17T02:03:59,375 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c6c0d7a0a7c0,45037,1731809032394, table=hbase:meta, region=1588230740 2024-11-17T02:03:59,375 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c6c0d7a0a7c0,45037,1731809032394, state=OPEN 2024-11-17T02:03:59,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,386 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,386 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,386 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,386 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN to state=OFFLINE, location=null, table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298 to restore RIT 2024-11-17T02:03:59,386 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN to state=OFFLINE, location=null, table=test, region=03b7adb3afa9bebd86dc1da38125b982 to restore RIT 2024-11-17T02:03:59,386 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN to state=OFFLINE, location=null, table=test, region=2f4fe90ec03839f5d26acd9d02569e56 to restore RIT 2024-11-17T02:03:59,387 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN to state=OFFLINE, location=null, table=test, region=25a67f85380b4a3cd6890ab81d0b3a27 to restore RIT 2024-11-17T02:03:59,387 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN to state=OFFLINE, location=null, table=test, region=de9924be32211cddceef43e5b4866dc8 to restore RIT 2024-11-17T02:03:59,387 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN to state=OFFLINE, location=null, table=test, region=1056118c93716b24d18f749826cf316a to restore RIT 2024-11-17T02:03:59,387 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN to state=OFFLINE, location=null, table=test, region=7eed07a820ab15d7af64462e2c1b70f7 to restore RIT 2024-11-17T02:03:59,387 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN to state=OFFLINE, location=null, table=test, region=112f5bbe15f72793585db9ba98950976 to restore RIT 2024-11-17T02:03:59,387 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731809069387 2024-11-17T02:03:59,387 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-17T02:03:59,387 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,388 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-17T02:03:59,390 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-17T02:03:59,390 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 25a67f85380b4a3cd6890ab81d0b3a27, server=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,390 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,390 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 25a67f85380b4a3cd6890ab81d0b3a27, server=c6c0d7a0a7c0,33127,1731809032263 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,395 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9924be32211cddceef43e5b4866dc8, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,395 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a17ac9e270344fa2752f51f8196693b2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,396 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81f322eb57dcb42c3b4269a738bafd6d, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,396 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15c64fab21a3ca03ec0a7afabc19cdc9, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,396 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8356c0fdb856c376b8678f2c2597565a, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,396 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=112f5bbe15f72793585db9ba98950976, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,396 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7517bce9e04879764302cccd45af20fb, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,396 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fd33da09f9d938215ac0e7127ad70d5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,397 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03b7adb3afa9bebd86dc1da38125b982, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,397 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05534a4a027a1fdc90227b7a2e681933, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,397 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac10d67ff6c3f527cc0861e738032bdb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,397 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1056118c93716b24d18f749826cf316a, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,397 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663bac36df6c9873ad776bb3ad42409f, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74108f0b014075d4993f0eee396e39e5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=25a67f85380b4a3cd6890ab81d0b3a27, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e1c1fc49350743a9cde50c433c8cb19, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=899f31eacba6d2d0aedbee7d8bbb4c9d, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7934444caa228b9e5d6d7b28b59a796, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6eba00b85aa16d5ccbfc8e3c35ff298, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4748cc4c5ad3838a3d0568cee797c169, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77bd97fde6bd169f3e20553b45a65e2c, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f4fe90ec03839f5d26acd9d02569e56, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db157dde5b6770afbf869f7733d95efb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d86e1fe125f78846dcaa5b9921c72db, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eed07a820ab15d7af64462e2c1b70f7, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=037b7a93730957425bc54697b3594bb2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,400 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-17T02:03:59,400 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731809099400 2024-11-17T02:03:59,400 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731809159400 2024-11-17T02:03:59,400 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-11-17T02:03:59,400 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-17T02:03:59,400 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,400 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-17T02:03:59,400 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-17T02:03:59,400 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-17T02:03:59,401 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-17T02:03:59,401 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-17T02:03:59,401 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-17T02:03:59,402 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-17T02:03:59,406 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-17T02:03:59,406 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,406 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,406 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,406 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,406 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,406 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,406 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,406 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,406 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,406 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,406 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,406 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,406 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-17T02:03:59,406 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-17T02:03:59,407 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,407 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,407 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-17T02:03:59,407 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-17T02:03:59,407 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-17T02:03:59,408 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-17T02:03:59,411 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true 2024-11-17T02:03:59,411 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,411 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,411 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T02:03:59,411 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,411 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,411 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,411 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN 2024-11-17T02:03:59,411 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN 2024-11-17T02:03:59,411 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN 2024-11-17T02:03:59,411 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN 2024-11-17T02:03:59,411 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN 2024-11-17T02:03:59,411 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN 2024-11-17T02:03:59,411 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN 2024-11-17T02:03:59,412 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN 2024-11-17T02:03:59,412 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2f4fe90ec03839f5d26acd9d02569e56, server=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,412 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure de9924be32211cddceef43e5b4866dc8, server=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,412 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,412 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 25a67f85380b4a3cd6890ab81d0b3a27, server=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,412 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,412 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,412 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,412 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,412 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,412 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,412 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,412 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,412 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN 2024-11-17T02:03:59,412 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,412 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN 2024-11-17T02:03:59,412 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,412 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN 2024-11-17T02:03:59,413 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,413 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN 2024-11-17T02:03:59,413 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,413 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN 2024-11-17T02:03:59,413 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,413 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN 2024-11-17T02:03:59,413 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,413 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN 2024-11-17T02:03:59,413 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,413 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN 2024-11-17T02:03:59,413 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2f4fe90ec03839f5d26acd9d02569e56, server=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,413 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure de9924be32211cddceef43e5b4866dc8, server=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,413 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,413 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 25a67f85380b4a3cd6890ab81d0b3a27, server=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,413 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-11-17T02:03:59,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=61 2024-11-17T02:03:59,415 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c6c0d7a0a7c0,45037,1731809032394, table=hbase:meta, region=1588230740 2024-11-17T02:03:59,415 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c6c0d7a0a7c0,45037,1731809032394, state=OPEN 2024-11-17T02:03:59,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,426 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,426 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,426 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,427 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN to state=OFFLINE, location=null, table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298 to restore RIT 2024-11-17T02:03:59,427 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN to state=OFFLINE, location=null, table=test, region=03b7adb3afa9bebd86dc1da38125b982 to restore RIT 2024-11-17T02:03:59,427 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN to state=OFFLINE, location=null, table=test, region=2f4fe90ec03839f5d26acd9d02569e56 to restore RIT 2024-11-17T02:03:59,427 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN to state=OFFLINE, location=null, table=test, region=25a67f85380b4a3cd6890ab81d0b3a27 to restore RIT 2024-11-17T02:03:59,427 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN to state=OFFLINE, location=null, table=test, region=de9924be32211cddceef43e5b4866dc8 to restore RIT 2024-11-17T02:03:59,427 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN to state=OFFLINE, location=null, table=test, region=1056118c93716b24d18f749826cf316a to restore RIT 2024-11-17T02:03:59,427 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN to state=OFFLINE, location=null, table=test, region=7eed07a820ab15d7af64462e2c1b70f7 to restore RIT 2024-11-17T02:03:59,427 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN to state=OFFLINE, location=null, table=test, region=112f5bbe15f72793585db9ba98950976 to restore RIT 2024-11-17T02:03:59,427 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731809069427 2024-11-17T02:03:59,428 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-17T02:03:59,428 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,429 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-17T02:03:59,430 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-17T02:03:59,430 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,430 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,430 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,435 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9924be32211cddceef43e5b4866dc8, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,435 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a17ac9e270344fa2752f51f8196693b2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,435 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81f322eb57dcb42c3b4269a738bafd6d, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,436 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15c64fab21a3ca03ec0a7afabc19cdc9, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,436 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8356c0fdb856c376b8678f2c2597565a, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,436 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=112f5bbe15f72793585db9ba98950976, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,436 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7517bce9e04879764302cccd45af20fb, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,437 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fd33da09f9d938215ac0e7127ad70d5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,437 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03b7adb3afa9bebd86dc1da38125b982, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,437 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05534a4a027a1fdc90227b7a2e681933, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,437 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac10d67ff6c3f527cc0861e738032bdb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,437 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1056118c93716b24d18f749826cf316a, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,437 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663bac36df6c9873ad776bb3ad42409f, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74108f0b014075d4993f0eee396e39e5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=25a67f85380b4a3cd6890ab81d0b3a27, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e1c1fc49350743a9cde50c433c8cb19, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=899f31eacba6d2d0aedbee7d8bbb4c9d, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7934444caa228b9e5d6d7b28b59a796, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6eba00b85aa16d5ccbfc8e3c35ff298, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4748cc4c5ad3838a3d0568cee797c169, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77bd97fde6bd169f3e20553b45a65e2c, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f4fe90ec03839f5d26acd9d02569e56, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db157dde5b6770afbf869f7733d95efb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d86e1fe125f78846dcaa5b9921c72db, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,440 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eed07a820ab15d7af64462e2c1b70f7, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,440 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=037b7a93730957425bc54697b3594bb2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,440 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-17T02:03:59,440 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731809099440 2024-11-17T02:03:59,440 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731809159440 2024-11-17T02:03:59,440 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-11-17T02:03:59,440 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-17T02:03:59,440 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,440 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-17T02:03:59,440 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-17T02:03:59,441 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-17T02:03:59,441 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-17T02:03:59,441 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-17T02:03:59,441 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-17T02:03:59,442 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-17T02:03:59,445 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-17T02:03:59,445 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,445 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,445 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,445 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,445 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,445 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,446 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,446 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,446 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,446 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,446 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,446 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-17T02:03:59,446 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-17T02:03:59,446 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,446 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,446 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-17T02:03:59,446 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-17T02:03:59,446 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-17T02:03:59,447 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-17T02:03:59,449 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true 2024-11-17T02:03:59,449 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,449 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,449 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T02:03:59,449 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,449 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,449 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,450 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN 2024-11-17T02:03:59,450 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN 2024-11-17T02:03:59,450 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN 2024-11-17T02:03:59,450 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN 2024-11-17T02:03:59,450 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN 2024-11-17T02:03:59,450 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN 2024-11-17T02:03:59,450 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN 2024-11-17T02:03:59,450 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN 2024-11-17T02:03:59,450 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2f4fe90ec03839f5d26acd9d02569e56, server=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,450 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure de9924be32211cddceef43e5b4866dc8, server=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,450 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,450 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,450 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,450 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,450 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,450 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,450 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,450 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,450 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,450 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN 2024-11-17T02:03:59,451 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,451 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN 2024-11-17T02:03:59,451 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,451 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN 2024-11-17T02:03:59,451 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,451 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN 2024-11-17T02:03:59,451 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,451 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN 2024-11-17T02:03:59,451 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,451 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN 2024-11-17T02:03:59,451 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,451 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN 2024-11-17T02:03:59,451 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,451 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN 2024-11-17T02:03:59,451 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2f4fe90ec03839f5d26acd9d02569e56, server=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,451 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure de9924be32211cddceef43e5b4866dc8, server=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,451 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=65, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,451 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-17T02:03:59,452 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=61 2024-11-17T02:03:59,452 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c6c0d7a0a7c0,45037,1731809032394, table=hbase:meta, region=1588230740 2024-11-17T02:03:59,453 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c6c0d7a0a7c0,45037,1731809032394, state=OPEN 2024-11-17T02:03:59,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,461 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,461 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,461 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,461 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN to state=OFFLINE, location=null, table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298 to restore RIT 2024-11-17T02:03:59,461 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN to state=OFFLINE, location=null, table=test, region=03b7adb3afa9bebd86dc1da38125b982 to restore RIT 2024-11-17T02:03:59,461 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN to state=OFFLINE, location=null, table=test, region=2f4fe90ec03839f5d26acd9d02569e56 to restore RIT 2024-11-17T02:03:59,462 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN to state=OFFLINE, location=null, table=test, region=25a67f85380b4a3cd6890ab81d0b3a27 to restore RIT 2024-11-17T02:03:59,462 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN to state=OFFLINE, location=null, table=test, region=de9924be32211cddceef43e5b4866dc8 to restore RIT 2024-11-17T02:03:59,462 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN to state=OFFLINE, location=null, table=test, region=1056118c93716b24d18f749826cf316a to restore RIT 2024-11-17T02:03:59,462 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN to state=OFFLINE, location=null, table=test, region=7eed07a820ab15d7af64462e2c1b70f7 to restore RIT 2024-11-17T02:03:59,462 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN to state=OFFLINE, location=null, table=test, region=112f5bbe15f72793585db9ba98950976 to restore RIT 2024-11-17T02:03:59,462 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731809069462 2024-11-17T02:03:59,462 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-17T02:03:59,462 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,464 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-17T02:03:59,465 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-17T02:03:59,465 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure de9924be32211cddceef43e5b4866dc8, server=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,465 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,465 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure de9924be32211cddceef43e5b4866dc8, server=c6c0d7a0a7c0,33127,1731809032263 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,469 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9924be32211cddceef43e5b4866dc8, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,469 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a17ac9e270344fa2752f51f8196693b2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,469 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81f322eb57dcb42c3b4269a738bafd6d, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,469 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15c64fab21a3ca03ec0a7afabc19cdc9, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,469 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8356c0fdb856c376b8678f2c2597565a, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=112f5bbe15f72793585db9ba98950976, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7517bce9e04879764302cccd45af20fb, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fd33da09f9d938215ac0e7127ad70d5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03b7adb3afa9bebd86dc1da38125b982, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05534a4a027a1fdc90227b7a2e681933, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac10d67ff6c3f527cc0861e738032bdb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1056118c93716b24d18f749826cf316a, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663bac36df6c9873ad776bb3ad42409f, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74108f0b014075d4993f0eee396e39e5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=25a67f85380b4a3cd6890ab81d0b3a27, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e1c1fc49350743a9cde50c433c8cb19, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=899f31eacba6d2d0aedbee7d8bbb4c9d, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7934444caa228b9e5d6d7b28b59a796, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6eba00b85aa16d5ccbfc8e3c35ff298, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4748cc4c5ad3838a3d0568cee797c169, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77bd97fde6bd169f3e20553b45a65e2c, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f4fe90ec03839f5d26acd9d02569e56, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db157dde5b6770afbf869f7733d95efb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d86e1fe125f78846dcaa5b9921c72db, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eed07a820ab15d7af64462e2c1b70f7, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=037b7a93730957425bc54697b3594bb2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,472 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-17T02:03:59,472 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731809099472 2024-11-17T02:03:59,472 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731809159472 2024-11-17T02:03:59,472 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-17T02:03:59,472 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-17T02:03:59,472 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,472 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-17T02:03:59,472 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-17T02:03:59,473 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-17T02:03:59,473 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-17T02:03:59,473 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-17T02:03:59,473 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-17T02:03:59,474 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-17T02:03:59,476 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-17T02:03:59,476 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,476 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,476 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,476 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32803 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,476 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,476 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,477 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,477 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,477 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,477 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,477 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,477 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-17T02:03:59,477 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32803 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,477 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-17T02:03:59,477 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,477 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,477 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-17T02:03:59,477 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-17T02:03:59,477 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-17T02:03:59,478 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-17T02:03:59,479 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true 2024-11-17T02:03:59,479 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,479 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,479 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T02:03:59,479 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,479 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,479 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,479 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN 2024-11-17T02:03:59,479 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN 2024-11-17T02:03:59,480 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN 2024-11-17T02:03:59,480 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN 2024-11-17T02:03:59,480 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN 2024-11-17T02:03:59,480 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN 2024-11-17T02:03:59,480 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN 2024-11-17T02:03:59,480 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN 2024-11-17T02:03:59,480 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2f4fe90ec03839f5d26acd9d02569e56, server=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,480 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure de9924be32211cddceef43e5b4866dc8, server=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,480 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,480 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,480 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,480 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,480 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,480 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,480 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,480 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,480 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN 2024-11-17T02:03:59,480 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,480 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN 2024-11-17T02:03:59,480 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,480 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN 2024-11-17T02:03:59,480 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,480 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN 2024-11-17T02:03:59,480 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,480 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN 2024-11-17T02:03:59,480 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,480 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN 2024-11-17T02:03:59,480 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,480 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN 2024-11-17T02:03:59,480 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,480 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN 2024-11-17T02:03:59,480 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2f4fe90ec03839f5d26acd9d02569e56, server=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,480 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure de9924be32211cddceef43e5b4866dc8, server=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,481 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-17T02:03:59,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=61 2024-11-17T02:03:59,481 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c6c0d7a0a7c0,45037,1731809032394, table=hbase:meta, region=1588230740 2024-11-17T02:03:59,482 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c6c0d7a0a7c0,45037,1731809032394, state=OPEN 2024-11-17T02:03:59,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,493 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,493 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,493 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,493 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN to state=OFFLINE, location=null, table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298 to restore RIT 2024-11-17T02:03:59,493 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN to state=OFFLINE, location=null, table=test, region=03b7adb3afa9bebd86dc1da38125b982 to restore RIT 2024-11-17T02:03:59,493 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN to state=OFFLINE, location=null, table=test, region=2f4fe90ec03839f5d26acd9d02569e56 to restore RIT 2024-11-17T02:03:59,494 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN to state=OFFLINE, location=null, table=test, region=25a67f85380b4a3cd6890ab81d0b3a27 to restore RIT 2024-11-17T02:03:59,494 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN to state=OFFLINE, location=null, table=test, region=de9924be32211cddceef43e5b4866dc8 to restore RIT 2024-11-17T02:03:59,494 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN to state=OFFLINE, location=null, table=test, region=1056118c93716b24d18f749826cf316a to restore RIT 2024-11-17T02:03:59,494 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN to state=OFFLINE, location=null, table=test, region=7eed07a820ab15d7af64462e2c1b70f7 to restore RIT 2024-11-17T02:03:59,494 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN to state=OFFLINE, location=null, table=test, region=112f5bbe15f72793585db9ba98950976 to restore RIT 2024-11-17T02:03:59,494 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731809069494 2024-11-17T02:03:59,494 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-17T02:03:59,494 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,496 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-17T02:03:59,497 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-17T02:03:59,497 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2f4fe90ec03839f5d26acd9d02569e56, server=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,497 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,497 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2f4fe90ec03839f5d26acd9d02569e56, server=c6c0d7a0a7c0,33127,1731809032263 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,500 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9924be32211cddceef43e5b4866dc8, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,500 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a17ac9e270344fa2752f51f8196693b2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,500 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81f322eb57dcb42c3b4269a738bafd6d, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,500 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15c64fab21a3ca03ec0a7afabc19cdc9, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,500 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8356c0fdb856c376b8678f2c2597565a, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,500 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=112f5bbe15f72793585db9ba98950976, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,500 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7517bce9e04879764302cccd45af20fb, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,501 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fd33da09f9d938215ac0e7127ad70d5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,501 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03b7adb3afa9bebd86dc1da38125b982, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,501 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05534a4a027a1fdc90227b7a2e681933, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,501 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac10d67ff6c3f527cc0861e738032bdb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,501 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1056118c93716b24d18f749826cf316a, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,501 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663bac36df6c9873ad776bb3ad42409f, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,501 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74108f0b014075d4993f0eee396e39e5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,501 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=25a67f85380b4a3cd6890ab81d0b3a27, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,501 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e1c1fc49350743a9cde50c433c8cb19, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,502 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=899f31eacba6d2d0aedbee7d8bbb4c9d, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,502 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7934444caa228b9e5d6d7b28b59a796, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,502 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6eba00b85aa16d5ccbfc8e3c35ff298, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,502 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4748cc4c5ad3838a3d0568cee797c169, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,502 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77bd97fde6bd169f3e20553b45a65e2c, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,502 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f4fe90ec03839f5d26acd9d02569e56, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,502 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db157dde5b6770afbf869f7733d95efb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,502 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d86e1fe125f78846dcaa5b9921c72db, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,502 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eed07a820ab15d7af64462e2c1b70f7, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,503 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=037b7a93730957425bc54697b3594bb2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,503 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-17T02:03:59,503 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731809099503 2024-11-17T02:03:59,503 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731809159503 2024-11-17T02:03:59,503 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-17T02:03:59,503 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-17T02:03:59,503 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,503 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-17T02:03:59,503 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-17T02:03:59,503 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-17T02:03:59,503 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-17T02:03:59,503 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-17T02:03:59,504 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-17T02:03:59,504 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-17T02:03:59,506 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-17T02:03:59,506 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,506 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,506 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,506 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,506 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,506 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,506 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,506 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,506 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,506 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-17T02:03:59,507 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-17T02:03:59,507 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,507 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,507 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-17T02:03:59,507 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-17T02:03:59,507 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-17T02:03:59,508 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-17T02:03:59,509 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true 2024-11-17T02:03:59,510 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,510 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,510 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T02:03:59,510 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,510 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,510 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,510 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN 2024-11-17T02:03:59,510 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN 2024-11-17T02:03:59,510 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN 2024-11-17T02:03:59,510 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN 2024-11-17T02:03:59,510 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN 2024-11-17T02:03:59,510 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN 2024-11-17T02:03:59,510 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN 2024-11-17T02:03:59,510 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN 2024-11-17T02:03:59,510 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2f4fe90ec03839f5d26acd9d02569e56, server=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,510 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,510 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,510 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,510 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,510 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,510 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,510 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,510 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,510 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN 2024-11-17T02:03:59,510 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,510 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN 2024-11-17T02:03:59,510 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,510 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN 2024-11-17T02:03:59,510 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,510 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN 2024-11-17T02:03:59,511 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,511 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN 2024-11-17T02:03:59,511 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,511 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN 2024-11-17T02:03:59,511 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,511 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN 2024-11-17T02:03:59,511 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,511 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN 2024-11-17T02:03:59,511 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2f4fe90ec03839f5d26acd9d02569e56, server=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,511 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-17T02:03:59,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=61 2024-11-17T02:03:59,512 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c6c0d7a0a7c0,45037,1731809032394, table=hbase:meta, region=1588230740 2024-11-17T02:03:59,512 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c6c0d7a0a7c0,45037,1731809032394, state=OPEN 2024-11-17T02:03:59,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,519 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,519 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,519 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,520 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN to state=OFFLINE, location=null, table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298 to restore RIT 2024-11-17T02:03:59,520 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN to state=OFFLINE, location=null, table=test, region=03b7adb3afa9bebd86dc1da38125b982 to restore RIT 2024-11-17T02:03:59,520 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN to state=OFFLINE, location=null, table=test, region=2f4fe90ec03839f5d26acd9d02569e56 to restore RIT 2024-11-17T02:03:59,520 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN to state=OFFLINE, location=null, table=test, region=25a67f85380b4a3cd6890ab81d0b3a27 to restore RIT 2024-11-17T02:03:59,520 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN to state=OFFLINE, location=null, table=test, region=de9924be32211cddceef43e5b4866dc8 to restore RIT 2024-11-17T02:03:59,520 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN to state=OFFLINE, location=null, table=test, region=1056118c93716b24d18f749826cf316a to restore RIT 2024-11-17T02:03:59,520 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN to state=OFFLINE, location=null, table=test, region=7eed07a820ab15d7af64462e2c1b70f7 to restore RIT 2024-11-17T02:03:59,520 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN to state=OFFLINE, location=null, table=test, region=112f5bbe15f72793585db9ba98950976 to restore RIT 2024-11-17T02:03:59,521 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731809069521 2024-11-17T02:03:59,521 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-17T02:03:59,521 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,522 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-17T02:03:59,523 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-17T02:03:59,523 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN 2024-11-17T02:03:59,523 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,523 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9924be32211cddceef43e5b4866dc8, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a17ac9e270344fa2752f51f8196693b2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81f322eb57dcb42c3b4269a738bafd6d, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,527 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15c64fab21a3ca03ec0a7afabc19cdc9, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,527 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8356c0fdb856c376b8678f2c2597565a, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,527 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=112f5bbe15f72793585db9ba98950976, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,527 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7517bce9e04879764302cccd45af20fb, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,527 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fd33da09f9d938215ac0e7127ad70d5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,527 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03b7adb3afa9bebd86dc1da38125b982, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,528 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05534a4a027a1fdc90227b7a2e681933, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,528 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac10d67ff6c3f527cc0861e738032bdb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,528 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1056118c93716b24d18f749826cf316a, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,528 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663bac36df6c9873ad776bb3ad42409f, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,528 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74108f0b014075d4993f0eee396e39e5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,528 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=25a67f85380b4a3cd6890ab81d0b3a27, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,528 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e1c1fc49350743a9cde50c433c8cb19, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,529 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=899f31eacba6d2d0aedbee7d8bbb4c9d, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,529 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7934444caa228b9e5d6d7b28b59a796, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,529 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6eba00b85aa16d5ccbfc8e3c35ff298, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,529 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4748cc4c5ad3838a3d0568cee797c169, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,529 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77bd97fde6bd169f3e20553b45a65e2c, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,529 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f4fe90ec03839f5d26acd9d02569e56, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,530 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db157dde5b6770afbf869f7733d95efb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,530 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d86e1fe125f78846dcaa5b9921c72db, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,530 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eed07a820ab15d7af64462e2c1b70f7, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,530 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=037b7a93730957425bc54697b3594bb2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,530 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-17T02:03:59,530 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731809099530 2024-11-17T02:03:59,530 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731809159530 2024-11-17T02:03:59,530 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-17T02:03:59,530 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-17T02:03:59,530 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,530 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-17T02:03:59,531 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-17T02:03:59,531 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-17T02:03:59,531 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-17T02:03:59,531 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-17T02:03:59,531 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-17T02:03:59,532 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-17T02:03:59,533 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-17T02:03:59,533 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,533 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,534 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,534 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,534 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,534 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,534 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,534 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,534 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-17T02:03:59,534 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-17T02:03:59,534 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,534 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,534 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-17T02:03:59,534 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-17T02:03:59,534 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-17T02:03:59,534 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-17T02:03:59,535 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true 2024-11-17T02:03:59,536 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,536 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,536 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T02:03:59,536 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,536 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,536 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,536 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN 2024-11-17T02:03:59,536 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN 2024-11-17T02:03:59,536 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN 2024-11-17T02:03:59,536 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN 2024-11-17T02:03:59,536 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN 2024-11-17T02:03:59,536 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN 2024-11-17T02:03:59,536 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN 2024-11-17T02:03:59,536 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN 2024-11-17T02:03:59,536 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,536 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,536 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,536 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,536 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,536 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,536 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,536 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,536 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN 2024-11-17T02:03:59,537 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,537 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN 2024-11-17T02:03:59,537 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,537 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN 2024-11-17T02:03:59,537 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,537 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN 2024-11-17T02:03:59,537 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,537 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN 2024-11-17T02:03:59,537 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,537 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN 2024-11-17T02:03:59,537 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,537 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN 2024-11-17T02:03:59,537 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,537 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN 2024-11-17T02:03:59,537 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-17T02:03:59,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=61 2024-11-17T02:03:59,538 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c6c0d7a0a7c0,45037,1731809032394, table=hbase:meta, region=1588230740 2024-11-17T02:03:59,538 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c6c0d7a0a7c0,45037,1731809032394, state=OPEN 2024-11-17T02:03:59,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,544 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,544 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,544 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,544 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN to state=OFFLINE, location=null, table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298 to restore RIT 2024-11-17T02:03:59,544 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN to state=OFFLINE, location=null, table=test, region=03b7adb3afa9bebd86dc1da38125b982 to restore RIT 2024-11-17T02:03:59,545 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN to state=OFFLINE, location=null, table=test, region=2f4fe90ec03839f5d26acd9d02569e56 to restore RIT 2024-11-17T02:03:59,545 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN to state=OFFLINE, location=null, table=test, region=25a67f85380b4a3cd6890ab81d0b3a27 to restore RIT 2024-11-17T02:03:59,545 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN to state=OFFLINE, location=null, table=test, region=de9924be32211cddceef43e5b4866dc8 to restore RIT 2024-11-17T02:03:59,545 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN to state=OFFLINE, location=null, table=test, region=1056118c93716b24d18f749826cf316a to restore RIT 2024-11-17T02:03:59,545 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN to state=OFFLINE, location=null, table=test, region=7eed07a820ab15d7af64462e2c1b70f7 to restore RIT 2024-11-17T02:03:59,545 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=112f5bbe15f72793585db9ba98950976, ASSIGN to state=OFFLINE, location=null, table=test, region=112f5bbe15f72793585db9ba98950976 to restore RIT 2024-11-17T02:03:59,545 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731809069545 2024-11-17T02:03:59,545 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-17T02:03:59,545 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,547 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-17T02:03:59,548 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-17T02:03:59,548 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN 2024-11-17T02:03:59,548 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,548 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,550 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9924be32211cddceef43e5b4866dc8, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,550 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a17ac9e270344fa2752f51f8196693b2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,550 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81f322eb57dcb42c3b4269a738bafd6d, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,551 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15c64fab21a3ca03ec0a7afabc19cdc9, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,551 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8356c0fdb856c376b8678f2c2597565a, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,551 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=112f5bbe15f72793585db9ba98950976, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,551 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7517bce9e04879764302cccd45af20fb, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,551 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fd33da09f9d938215ac0e7127ad70d5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,551 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03b7adb3afa9bebd86dc1da38125b982, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,552 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05534a4a027a1fdc90227b7a2e681933, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,552 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac10d67ff6c3f527cc0861e738032bdb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,552 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1056118c93716b24d18f749826cf316a, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,552 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663bac36df6c9873ad776bb3ad42409f, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,552 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74108f0b014075d4993f0eee396e39e5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,553 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=25a67f85380b4a3cd6890ab81d0b3a27, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,553 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e1c1fc49350743a9cde50c433c8cb19, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,553 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=899f31eacba6d2d0aedbee7d8bbb4c9d, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,553 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7934444caa228b9e5d6d7b28b59a796, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,553 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6eba00b85aa16d5ccbfc8e3c35ff298, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,553 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4748cc4c5ad3838a3d0568cee797c169, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,553 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77bd97fde6bd169f3e20553b45a65e2c, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f4fe90ec03839f5d26acd9d02569e56, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db157dde5b6770afbf869f7733d95efb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d86e1fe125f78846dcaa5b9921c72db, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eed07a820ab15d7af64462e2c1b70f7, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=037b7a93730957425bc54697b3594bb2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,554 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-17T02:03:59,554 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731809099554 2024-11-17T02:03:59,554 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731809159554 2024-11-17T02:03:59,555 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-17T02:03:59,555 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-17T02:03:59,555 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,555 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-17T02:03:59,555 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-17T02:03:59,555 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-17T02:03:59,555 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-17T02:03:59,555 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-17T02:03:59,556 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-17T02:03:59,557 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-17T02:03:59,559 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-17T02:03:59,559 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,559 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,559 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,559 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,559 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,559 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,559 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,559 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-17T02:03:59,559 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-17T02:03:59,559 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,559 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,559 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-17T02:03:59,560 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-17T02:03:59,560 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-17T02:03:59,561 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-17T02:03:59,563 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true 2024-11-17T02:03:59,563 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,563 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,563 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T02:03:59,563 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,563 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,563 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,563 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN 2024-11-17T02:03:59,563 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN 2024-11-17T02:03:59,563 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN 2024-11-17T02:03:59,563 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN 2024-11-17T02:03:59,563 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN 2024-11-17T02:03:59,563 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN 2024-11-17T02:03:59,563 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN 2024-11-17T02:03:59,564 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,564 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,564 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,564 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,564 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,564 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,564 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,564 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,564 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN 2024-11-17T02:03:59,564 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,564 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN 2024-11-17T02:03:59,564 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,564 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN 2024-11-17T02:03:59,564 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,564 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN 2024-11-17T02:03:59,564 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,564 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN 2024-11-17T02:03:59,564 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,564 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN 2024-11-17T02:03:59,564 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,564 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN 2024-11-17T02:03:59,564 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-17T02:03:59,565 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=61 2024-11-17T02:03:59,565 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c6c0d7a0a7c0,45037,1731809032394, table=hbase:meta, region=1588230740 2024-11-17T02:03:59,566 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c6c0d7a0a7c0,45037,1731809032394, state=OPEN 2024-11-17T02:03:59,577 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,577 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,577 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,577 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,577 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,577 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,578 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN to state=OFFLINE, location=null, table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298 to restore RIT 2024-11-17T02:03:59,578 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN to state=OFFLINE, location=null, table=test, region=03b7adb3afa9bebd86dc1da38125b982 to restore RIT 2024-11-17T02:03:59,578 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN to state=OFFLINE, location=null, table=test, region=2f4fe90ec03839f5d26acd9d02569e56 to restore RIT 2024-11-17T02:03:59,578 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN to state=OFFLINE, location=null, table=test, region=25a67f85380b4a3cd6890ab81d0b3a27 to restore RIT 2024-11-17T02:03:59,578 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN to state=OFFLINE, location=null, table=test, region=de9924be32211cddceef43e5b4866dc8 to restore RIT 2024-11-17T02:03:59,578 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN to state=OFFLINE, location=null, table=test, region=1056118c93716b24d18f749826cf316a to restore RIT 2024-11-17T02:03:59,578 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7eed07a820ab15d7af64462e2c1b70f7, ASSIGN to state=OFFLINE, location=null, table=test, region=7eed07a820ab15d7af64462e2c1b70f7 to restore RIT 2024-11-17T02:03:59,579 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731809069579 2024-11-17T02:03:59,579 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-17T02:03:59,579 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,581 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-17T02:03:59,582 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-17T02:03:59,582 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN 2024-11-17T02:03:59,582 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,582 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9924be32211cddceef43e5b4866dc8, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a17ac9e270344fa2752f51f8196693b2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81f322eb57dcb42c3b4269a738bafd6d, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15c64fab21a3ca03ec0a7afabc19cdc9, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8356c0fdb856c376b8678f2c2597565a, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=112f5bbe15f72793585db9ba98950976, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7517bce9e04879764302cccd45af20fb, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fd33da09f9d938215ac0e7127ad70d5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03b7adb3afa9bebd86dc1da38125b982, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05534a4a027a1fdc90227b7a2e681933, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac10d67ff6c3f527cc0861e738032bdb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1056118c93716b24d18f749826cf316a, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663bac36df6c9873ad776bb3ad42409f, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74108f0b014075d4993f0eee396e39e5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=25a67f85380b4a3cd6890ab81d0b3a27, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e1c1fc49350743a9cde50c433c8cb19, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=899f31eacba6d2d0aedbee7d8bbb4c9d, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7934444caa228b9e5d6d7b28b59a796, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6eba00b85aa16d5ccbfc8e3c35ff298, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4748cc4c5ad3838a3d0568cee797c169, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77bd97fde6bd169f3e20553b45a65e2c, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f4fe90ec03839f5d26acd9d02569e56, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db157dde5b6770afbf869f7733d95efb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d86e1fe125f78846dcaa5b9921c72db, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eed07a820ab15d7af64462e2c1b70f7, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=037b7a93730957425bc54697b3594bb2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,588 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-17T02:03:59,589 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731809099588 2024-11-17T02:03:59,589 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731809159589 2024-11-17T02:03:59,589 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-17T02:03:59,589 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-17T02:03:59,589 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,589 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-17T02:03:59,589 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-17T02:03:59,589 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-17T02:03:59,589 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-17T02:03:59,589 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-17T02:03:59,590 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-17T02:03:59,591 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-17T02:03:59,593 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-17T02:03:59,593 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,593 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,593 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,593 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,593 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,593 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,593 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-17T02:03:59,593 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-17T02:03:59,593 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,593 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,593 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-17T02:03:59,593 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-17T02:03:59,593 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-17T02:03:59,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-17T02:03:59,596 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true 2024-11-17T02:03:59,596 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,596 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,596 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T02:03:59,596 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,596 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,596 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,596 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN 2024-11-17T02:03:59,596 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN 2024-11-17T02:03:59,596 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN 2024-11-17T02:03:59,596 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN 2024-11-17T02:03:59,596 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN 2024-11-17T02:03:59,596 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN 2024-11-17T02:03:59,596 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,596 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,596 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,597 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,597 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,597 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,597 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,597 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,597 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN 2024-11-17T02:03:59,597 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,597 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN 2024-11-17T02:03:59,597 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,597 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN 2024-11-17T02:03:59,597 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,597 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN 2024-11-17T02:03:59,597 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,597 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN 2024-11-17T02:03:59,597 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,597 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN 2024-11-17T02:03:59,597 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-17T02:03:59,598 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=61 2024-11-17T02:03:59,598 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c6c0d7a0a7c0,45037,1731809032394, table=hbase:meta, region=1588230740 2024-11-17T02:03:59,599 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c6c0d7a0a7c0,45037,1731809032394, state=OPEN 2024-11-17T02:03:59,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,609 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,609 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,609 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,610 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN to state=OFFLINE, location=null, table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298 to restore RIT 2024-11-17T02:03:59,610 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN to state=OFFLINE, location=null, table=test, region=03b7adb3afa9bebd86dc1da38125b982 to restore RIT 2024-11-17T02:03:59,610 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN to state=OFFLINE, location=null, table=test, region=2f4fe90ec03839f5d26acd9d02569e56 to restore RIT 2024-11-17T02:03:59,610 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN to state=OFFLINE, location=null, table=test, region=25a67f85380b4a3cd6890ab81d0b3a27 to restore RIT 2024-11-17T02:03:59,610 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN to state=OFFLINE, location=null, table=test, region=de9924be32211cddceef43e5b4866dc8 to restore RIT 2024-11-17T02:03:59,610 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1056118c93716b24d18f749826cf316a, ASSIGN to state=OFFLINE, location=null, table=test, region=1056118c93716b24d18f749826cf316a to restore RIT 2024-11-17T02:03:59,611 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731809069611 2024-11-17T02:03:59,611 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-17T02:03:59,611 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,612 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-17T02:03:59,615 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-17T02:03:59,615 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN 2024-11-17T02:03:59,615 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,615 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,616 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9924be32211cddceef43e5b4866dc8, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,616 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a17ac9e270344fa2752f51f8196693b2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81f322eb57dcb42c3b4269a738bafd6d, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15c64fab21a3ca03ec0a7afabc19cdc9, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8356c0fdb856c376b8678f2c2597565a, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=112f5bbe15f72793585db9ba98950976, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7517bce9e04879764302cccd45af20fb, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fd33da09f9d938215ac0e7127ad70d5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03b7adb3afa9bebd86dc1da38125b982, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05534a4a027a1fdc90227b7a2e681933, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac10d67ff6c3f527cc0861e738032bdb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1056118c93716b24d18f749826cf316a, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663bac36df6c9873ad776bb3ad42409f, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74108f0b014075d4993f0eee396e39e5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=25a67f85380b4a3cd6890ab81d0b3a27, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e1c1fc49350743a9cde50c433c8cb19, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=899f31eacba6d2d0aedbee7d8bbb4c9d, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7934444caa228b9e5d6d7b28b59a796, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6eba00b85aa16d5ccbfc8e3c35ff298, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4748cc4c5ad3838a3d0568cee797c169, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77bd97fde6bd169f3e20553b45a65e2c, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f4fe90ec03839f5d26acd9d02569e56, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db157dde5b6770afbf869f7733d95efb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d86e1fe125f78846dcaa5b9921c72db, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eed07a820ab15d7af64462e2c1b70f7, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=037b7a93730957425bc54697b3594bb2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,619 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-17T02:03:59,619 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731809099619 2024-11-17T02:03:59,619 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731809159619 2024-11-17T02:03:59,619 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-17T02:03:59,619 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-17T02:03:59,619 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,620 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-17T02:03:59,620 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-17T02:03:59,620 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-17T02:03:59,620 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-17T02:03:59,620 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-17T02:03:59,620 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-17T02:03:59,621 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-17T02:03:59,623 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-17T02:03:59,623 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,623 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,623 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,623 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,623 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,623 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-17T02:03:59,623 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-17T02:03:59,623 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,623 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,623 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-17T02:03:59,623 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-17T02:03:59,623 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-17T02:03:59,624 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-17T02:03:59,625 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true 2024-11-17T02:03:59,625 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,625 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,625 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T02:03:59,625 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,625 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,625 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,625 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN 2024-11-17T02:03:59,625 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN 2024-11-17T02:03:59,625 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN 2024-11-17T02:03:59,625 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN 2024-11-17T02:03:59,625 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN 2024-11-17T02:03:59,625 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,625 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,625 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,625 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,625 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,625 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,625 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,625 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,625 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN 2024-11-17T02:03:59,625 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,625 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN 2024-11-17T02:03:59,625 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,625 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN 2024-11-17T02:03:59,625 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,625 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN 2024-11-17T02:03:59,625 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,625 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN 2024-11-17T02:03:59,626 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-17T02:03:59,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=61 2024-11-17T02:03:59,627 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c6c0d7a0a7c0,45037,1731809032394, table=hbase:meta, region=1588230740 2024-11-17T02:03:59,627 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c6c0d7a0a7c0,45037,1731809032394, state=OPEN 2024-11-17T02:03:59,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,636 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,636 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,636 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,636 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN to state=OFFLINE, location=null, table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298 to restore RIT 2024-11-17T02:03:59,636 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN to state=OFFLINE, location=null, table=test, region=03b7adb3afa9bebd86dc1da38125b982 to restore RIT 2024-11-17T02:03:59,636 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN to state=OFFLINE, location=null, table=test, region=2f4fe90ec03839f5d26acd9d02569e56 to restore RIT 2024-11-17T02:03:59,636 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN to state=OFFLINE, location=null, table=test, region=25a67f85380b4a3cd6890ab81d0b3a27 to restore RIT 2024-11-17T02:03:59,636 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de9924be32211cddceef43e5b4866dc8, ASSIGN to state=OFFLINE, location=null, table=test, region=de9924be32211cddceef43e5b4866dc8 to restore RIT 2024-11-17T02:03:59,637 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731809069637 2024-11-17T02:03:59,637 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-17T02:03:59,637 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,638 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-17T02:03:59,639 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-17T02:03:59,639 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN 2024-11-17T02:03:59,639 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,639 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,641 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9924be32211cddceef43e5b4866dc8, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,642 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a17ac9e270344fa2752f51f8196693b2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,642 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81f322eb57dcb42c3b4269a738bafd6d, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,642 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15c64fab21a3ca03ec0a7afabc19cdc9, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,642 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8356c0fdb856c376b8678f2c2597565a, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,643 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=112f5bbe15f72793585db9ba98950976, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,643 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7517bce9e04879764302cccd45af20fb, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,643 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fd33da09f9d938215ac0e7127ad70d5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,643 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03b7adb3afa9bebd86dc1da38125b982, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,643 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05534a4a027a1fdc90227b7a2e681933, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,643 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac10d67ff6c3f527cc0861e738032bdb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,643 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1056118c93716b24d18f749826cf316a, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,644 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663bac36df6c9873ad776bb3ad42409f, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,644 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74108f0b014075d4993f0eee396e39e5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,644 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=25a67f85380b4a3cd6890ab81d0b3a27, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,644 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e1c1fc49350743a9cde50c433c8cb19, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,644 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=899f31eacba6d2d0aedbee7d8bbb4c9d, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,644 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7934444caa228b9e5d6d7b28b59a796, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,644 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6eba00b85aa16d5ccbfc8e3c35ff298, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,645 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4748cc4c5ad3838a3d0568cee797c169, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,645 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77bd97fde6bd169f3e20553b45a65e2c, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,645 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f4fe90ec03839f5d26acd9d02569e56, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,645 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db157dde5b6770afbf869f7733d95efb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,645 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d86e1fe125f78846dcaa5b9921c72db, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,645 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eed07a820ab15d7af64462e2c1b70f7, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,645 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=037b7a93730957425bc54697b3594bb2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,645 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-17T02:03:59,646 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731809099646 2024-11-17T02:03:59,646 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731809159646 2024-11-17T02:03:59,646 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-17T02:03:59,646 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-17T02:03:59,646 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,646 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-17T02:03:59,646 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-17T02:03:59,646 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-17T02:03:59,646 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-17T02:03:59,646 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-17T02:03:59,647 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-17T02:03:59,648 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-17T02:03:59,650 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-17T02:03:59,650 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,650 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,650 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,650 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,650 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-17T02:03:59,650 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-17T02:03:59,650 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,650 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,650 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-17T02:03:59,650 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-17T02:03:59,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-17T02:03:59,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-17T02:03:59,653 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true 2024-11-17T02:03:59,653 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,653 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,653 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T02:03:59,653 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,653 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,653 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,653 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN 2024-11-17T02:03:59,653 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN 2024-11-17T02:03:59,653 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN 2024-11-17T02:03:59,653 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN 2024-11-17T02:03:59,653 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,653 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,653 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,653 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,653 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,654 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,654 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,654 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,654 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN 2024-11-17T02:03:59,654 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,654 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN 2024-11-17T02:03:59,654 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,654 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN 2024-11-17T02:03:59,654 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,654 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN 2024-11-17T02:03:59,654 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-17T02:03:59,655 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=61 2024-11-17T02:03:59,655 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c6c0d7a0a7c0,45037,1731809032394, table=hbase:meta, region=1588230740 2024-11-17T02:03:59,655 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c6c0d7a0a7c0,45037,1731809032394, state=OPEN 2024-11-17T02:03:59,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,661 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,661 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,661 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,661 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN to state=OFFLINE, location=null, table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298 to restore RIT 2024-11-17T02:03:59,661 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN to state=OFFLINE, location=null, table=test, region=03b7adb3afa9bebd86dc1da38125b982 to restore RIT 2024-11-17T02:03:59,661 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN to state=OFFLINE, location=null, table=test, region=2f4fe90ec03839f5d26acd9d02569e56 to restore RIT 2024-11-17T02:03:59,662 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=25a67f85380b4a3cd6890ab81d0b3a27, ASSIGN to state=OFFLINE, location=null, table=test, region=25a67f85380b4a3cd6890ab81d0b3a27 to restore RIT 2024-11-17T02:03:59,662 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731809069662 2024-11-17T02:03:59,662 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-17T02:03:59,662 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,663 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-17T02:03:59,665 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-17T02:03:59,665 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN 2024-11-17T02:03:59,665 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,665 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,667 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9924be32211cddceef43e5b4866dc8, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,667 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a17ac9e270344fa2752f51f8196693b2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,667 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81f322eb57dcb42c3b4269a738bafd6d, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15c64fab21a3ca03ec0a7afabc19cdc9, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8356c0fdb856c376b8678f2c2597565a, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=112f5bbe15f72793585db9ba98950976, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7517bce9e04879764302cccd45af20fb, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fd33da09f9d938215ac0e7127ad70d5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03b7adb3afa9bebd86dc1da38125b982, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05534a4a027a1fdc90227b7a2e681933, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac10d67ff6c3f527cc0861e738032bdb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1056118c93716b24d18f749826cf316a, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663bac36df6c9873ad776bb3ad42409f, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74108f0b014075d4993f0eee396e39e5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=25a67f85380b4a3cd6890ab81d0b3a27, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e1c1fc49350743a9cde50c433c8cb19, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=899f31eacba6d2d0aedbee7d8bbb4c9d, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7934444caa228b9e5d6d7b28b59a796, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6eba00b85aa16d5ccbfc8e3c35ff298, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4748cc4c5ad3838a3d0568cee797c169, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77bd97fde6bd169f3e20553b45a65e2c, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f4fe90ec03839f5d26acd9d02569e56, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db157dde5b6770afbf869f7733d95efb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d86e1fe125f78846dcaa5b9921c72db, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eed07a820ab15d7af64462e2c1b70f7, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=037b7a93730957425bc54697b3594bb2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,670 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-17T02:03:59,670 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731809099670 2024-11-17T02:03:59,670 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731809159670 2024-11-17T02:03:59,670 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-17T02:03:59,670 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-17T02:03:59,670 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,670 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-17T02:03:59,670 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-17T02:03:59,671 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-17T02:03:59,671 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-17T02:03:59,671 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-17T02:03:59,671 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-17T02:03:59,672 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-17T02:03:59,673 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-17T02:03:59,673 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,673 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,673 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,673 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-17T02:03:59,673 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-17T02:03:59,673 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,673 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,673 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-17T02:03:59,673 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-17T02:03:59,674 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-17T02:03:59,674 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-17T02:03:59,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true 2024-11-17T02:03:59,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T02:03:59,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,676 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN 2024-11-17T02:03:59,676 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN 2024-11-17T02:03:59,676 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN 2024-11-17T02:03:59,676 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,676 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,676 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,676 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,676 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,676 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,676 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,676 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,676 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN 2024-11-17T02:03:59,676 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,676 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN 2024-11-17T02:03:59,676 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,676 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN 2024-11-17T02:03:59,676 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-17T02:03:59,677 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=61 2024-11-17T02:03:59,677 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c6c0d7a0a7c0,45037,1731809032394, table=hbase:meta, region=1588230740 2024-11-17T02:03:59,677 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c6c0d7a0a7c0,45037,1731809032394, state=OPEN 2024-11-17T02:03:59,682 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32803 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,683 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32803 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,686 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,686 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,686 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,686 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN to state=OFFLINE, location=null, table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298 to restore RIT 2024-11-17T02:03:59,686 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN to state=OFFLINE, location=null, table=test, region=03b7adb3afa9bebd86dc1da38125b982 to restore RIT 2024-11-17T02:03:59,687 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2f4fe90ec03839f5d26acd9d02569e56, ASSIGN to state=OFFLINE, location=null, table=test, region=2f4fe90ec03839f5d26acd9d02569e56 to restore RIT 2024-11-17T02:03:59,687 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731809069687 2024-11-17T02:03:59,687 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-17T02:03:59,687 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,688 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-17T02:03:59,689 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-17T02:03:59,689 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN 2024-11-17T02:03:59,689 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,689 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9924be32211cddceef43e5b4866dc8, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a17ac9e270344fa2752f51f8196693b2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81f322eb57dcb42c3b4269a738bafd6d, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15c64fab21a3ca03ec0a7afabc19cdc9, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8356c0fdb856c376b8678f2c2597565a, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=112f5bbe15f72793585db9ba98950976, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7517bce9e04879764302cccd45af20fb, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fd33da09f9d938215ac0e7127ad70d5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03b7adb3afa9bebd86dc1da38125b982, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05534a4a027a1fdc90227b7a2e681933, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac10d67ff6c3f527cc0861e738032bdb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1056118c93716b24d18f749826cf316a, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663bac36df6c9873ad776bb3ad42409f, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74108f0b014075d4993f0eee396e39e5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=25a67f85380b4a3cd6890ab81d0b3a27, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e1c1fc49350743a9cde50c433c8cb19, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=899f31eacba6d2d0aedbee7d8bbb4c9d, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7934444caa228b9e5d6d7b28b59a796, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6eba00b85aa16d5ccbfc8e3c35ff298, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4748cc4c5ad3838a3d0568cee797c169, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77bd97fde6bd169f3e20553b45a65e2c, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f4fe90ec03839f5d26acd9d02569e56, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db157dde5b6770afbf869f7733d95efb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d86e1fe125f78846dcaa5b9921c72db, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eed07a820ab15d7af64462e2c1b70f7, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=037b7a93730957425bc54697b3594bb2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,695 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-17T02:03:59,695 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731809099695 2024-11-17T02:03:59,695 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731809159695 2024-11-17T02:03:59,695 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-17T02:03:59,695 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-17T02:03:59,695 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,695 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-17T02:03:59,695 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-17T02:03:59,695 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-17T02:03:59,695 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-17T02:03:59,695 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-17T02:03:59,696 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-17T02:03:59,696 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-17T02:03:59,698 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-17T02:03:59,698 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,698 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,698 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-17T02:03:59,698 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-17T02:03:59,698 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,698 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,699 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-17T02:03:59,699 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-17T02:03:59,699 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-17T02:03:59,699 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-17T02:03:59,700 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true 2024-11-17T02:03:59,700 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,700 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,700 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T02:03:59,700 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,700 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,700 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,700 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN 2024-11-17T02:03:59,700 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN 2024-11-17T02:03:59,700 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,701 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,701 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,701 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,701 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,701 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,701 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,701 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,701 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN 2024-11-17T02:03:59,701 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,701 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN 2024-11-17T02:03:59,701 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-17T02:03:59,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=61 2024-11-17T02:03:59,702 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c6c0d7a0a7c0,45037,1731809032394, table=hbase:meta, region=1588230740 2024-11-17T02:03:59,702 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c6c0d7a0a7c0,45037,1731809032394, state=OPEN 2024-11-17T02:03:59,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,732 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,732 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,732 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,732 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN to state=OFFLINE, location=null, table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298 to restore RIT 2024-11-17T02:03:59,733 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03b7adb3afa9bebd86dc1da38125b982, ASSIGN to state=OFFLINE, location=null, table=test, region=03b7adb3afa9bebd86dc1da38125b982 to restore RIT 2024-11-17T02:03:59,734 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731809069734 2024-11-17T02:03:59,734 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-17T02:03:59,734 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,735 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-17T02:03:59,736 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-17T02:03:59,736 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN 2024-11-17T02:03:59,736 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,736 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9924be32211cddceef43e5b4866dc8, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a17ac9e270344fa2752f51f8196693b2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81f322eb57dcb42c3b4269a738bafd6d, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15c64fab21a3ca03ec0a7afabc19cdc9, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8356c0fdb856c376b8678f2c2597565a, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=112f5bbe15f72793585db9ba98950976, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7517bce9e04879764302cccd45af20fb, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fd33da09f9d938215ac0e7127ad70d5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03b7adb3afa9bebd86dc1da38125b982, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05534a4a027a1fdc90227b7a2e681933, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,740 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac10d67ff6c3f527cc0861e738032bdb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,740 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1056118c93716b24d18f749826cf316a, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,740 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663bac36df6c9873ad776bb3ad42409f, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,740 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74108f0b014075d4993f0eee396e39e5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,740 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=25a67f85380b4a3cd6890ab81d0b3a27, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,740 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e1c1fc49350743a9cde50c433c8cb19, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,740 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=899f31eacba6d2d0aedbee7d8bbb4c9d, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,740 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7934444caa228b9e5d6d7b28b59a796, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,740 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6eba00b85aa16d5ccbfc8e3c35ff298, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,740 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4748cc4c5ad3838a3d0568cee797c169, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,740 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77bd97fde6bd169f3e20553b45a65e2c, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,740 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f4fe90ec03839f5d26acd9d02569e56, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,740 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db157dde5b6770afbf869f7733d95efb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,741 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d86e1fe125f78846dcaa5b9921c72db, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,741 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eed07a820ab15d7af64462e2c1b70f7, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,741 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=037b7a93730957425bc54697b3594bb2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,741 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-17T02:03:59,741 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731809099741 2024-11-17T02:03:59,741 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731809159741 2024-11-17T02:03:59,741 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-17T02:03:59,741 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-17T02:03:59,741 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,741 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-17T02:03:59,741 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-17T02:03:59,741 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-17T02:03:59,741 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-17T02:03:59,741 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-17T02:03:59,742 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-17T02:03:59,742 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-17T02:03:59,744 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-17T02:03:59,744 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,744 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-17T02:03:59,744 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-17T02:03:59,744 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,744 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,744 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-17T02:03:59,744 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-17T02:03:59,745 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-17T02:03:59,745 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-17T02:03:59,746 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true 2024-11-17T02:03:59,746 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,746 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,746 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T02:03:59,746 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,746 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,746 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,746 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN 2024-11-17T02:03:59,746 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,746 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,746 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,746 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,746 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,746 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,746 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,746 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,746 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN 2024-11-17T02:03:59,747 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-17T02:03:59,747 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=61 2024-11-17T02:03:59,747 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c6c0d7a0a7c0,45037,1731809032394, table=hbase:meta, region=1588230740 2024-11-17T02:03:59,747 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c6c0d7a0a7c0,45037,1731809032394, state=OPEN 2024-11-17T02:03:59,761 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,761 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,761 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,761 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,761 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,761 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,761 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298, ASSIGN to state=OFFLINE, location=null, table=test, region=b6eba00b85aa16d5ccbfc8e3c35ff298 to restore RIT 2024-11-17T02:03:59,762 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731809069762 2024-11-17T02:03:59,762 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-17T02:03:59,762 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,763 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-17T02:03:59,764 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-17T02:03:59,764 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,764 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,764 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9924be32211cddceef43e5b4866dc8, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,767 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a17ac9e270344fa2752f51f8196693b2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,767 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81f322eb57dcb42c3b4269a738bafd6d, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,767 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15c64fab21a3ca03ec0a7afabc19cdc9, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,767 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8356c0fdb856c376b8678f2c2597565a, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,767 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=112f5bbe15f72793585db9ba98950976, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,767 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7517bce9e04879764302cccd45af20fb, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,767 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fd33da09f9d938215ac0e7127ad70d5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,767 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03b7adb3afa9bebd86dc1da38125b982, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,767 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05534a4a027a1fdc90227b7a2e681933, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,768 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac10d67ff6c3f527cc0861e738032bdb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,768 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1056118c93716b24d18f749826cf316a, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,768 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663bac36df6c9873ad776bb3ad42409f, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,768 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74108f0b014075d4993f0eee396e39e5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,768 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=25a67f85380b4a3cd6890ab81d0b3a27, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,768 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e1c1fc49350743a9cde50c433c8cb19, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,768 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=899f31eacba6d2d0aedbee7d8bbb4c9d, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,768 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7934444caa228b9e5d6d7b28b59a796, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,768 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6eba00b85aa16d5ccbfc8e3c35ff298, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,768 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4748cc4c5ad3838a3d0568cee797c169, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,769 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77bd97fde6bd169f3e20553b45a65e2c, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,769 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f4fe90ec03839f5d26acd9d02569e56, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,769 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db157dde5b6770afbf869f7733d95efb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,769 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d86e1fe125f78846dcaa5b9921c72db, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,769 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eed07a820ab15d7af64462e2c1b70f7, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,769 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=037b7a93730957425bc54697b3594bb2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,769 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-17T02:03:59,769 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731809099769 2024-11-17T02:03:59,769 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731809159769 2024-11-17T02:03:59,769 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-17T02:03:59,769 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-17T02:03:59,769 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,769 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-17T02:03:59,769 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-17T02:03:59,770 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-17T02:03:59,770 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-17T02:03:59,770 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-17T02:03:59,770 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-17T02:03:59,770 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-17T02:03:59,772 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-17T02:03:59,772 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-17T02:03:59,772 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-17T02:03:59,772 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,772 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,772 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-17T02:03:59,772 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-17T02:03:59,772 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-17T02:03:59,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-17T02:03:59,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true 2024-11-17T02:03:59,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T02:03:59,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,774 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,774 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-17T02:03:59,775 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=61 2024-11-17T02:03:59,775 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c6c0d7a0a7c0,45037,1731809032394, table=hbase:meta, region=1588230740 2024-11-17T02:03:59,775 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c6c0d7a0a7c0,45037,1731809032394, state=OPEN 2024-11-17T02:03:59,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,786 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,786 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,786 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,787 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731809069787 2024-11-17T02:03:59,787 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-17T02:03:59,787 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,789 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-17T02:03:59,791 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-17T02:03:59,791 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,791 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,792 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9924be32211cddceef43e5b4866dc8, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,792 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a17ac9e270344fa2752f51f8196693b2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,792 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81f322eb57dcb42c3b4269a738bafd6d, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,792 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15c64fab21a3ca03ec0a7afabc19cdc9, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,792 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8356c0fdb856c376b8678f2c2597565a, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,792 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=112f5bbe15f72793585db9ba98950976, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,792 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7517bce9e04879764302cccd45af20fb, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,792 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fd33da09f9d938215ac0e7127ad70d5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,792 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03b7adb3afa9bebd86dc1da38125b982, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,792 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05534a4a027a1fdc90227b7a2e681933, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,793 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac10d67ff6c3f527cc0861e738032bdb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,793 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1056118c93716b24d18f749826cf316a, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,793 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663bac36df6c9873ad776bb3ad42409f, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,793 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74108f0b014075d4993f0eee396e39e5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,793 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=25a67f85380b4a3cd6890ab81d0b3a27, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,793 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e1c1fc49350743a9cde50c433c8cb19, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,793 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=899f31eacba6d2d0aedbee7d8bbb4c9d, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,793 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7934444caa228b9e5d6d7b28b59a796, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,793 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6eba00b85aa16d5ccbfc8e3c35ff298, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,793 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4748cc4c5ad3838a3d0568cee797c169, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,793 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77bd97fde6bd169f3e20553b45a65e2c, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,793 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f4fe90ec03839f5d26acd9d02569e56, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,793 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db157dde5b6770afbf869f7733d95efb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,794 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d86e1fe125f78846dcaa5b9921c72db, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,794 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eed07a820ab15d7af64462e2c1b70f7, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,794 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=037b7a93730957425bc54697b3594bb2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,794 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-17T02:03:59,794 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731809099794 2024-11-17T02:03:59,794 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731809159794 2024-11-17T02:03:59,794 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-17T02:03:59,794 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-17T02:03:59,794 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,794 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-17T02:03:59,794 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-17T02:03:59,794 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-17T02:03:59,794 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-17T02:03:59,794 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-17T02:03:59,795 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-17T02:03:59,795 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-17T02:03:59,797 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-17T02:03:59,797 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-17T02:03:59,797 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-17T02:03:59,797 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,797 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,797 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-17T02:03:59,797 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-17T02:03:59,797 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-17T02:03:59,798 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true 2024-11-17T02:03:59,798 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,798 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,798 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T02:03:59,798 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,798 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,798 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,798 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,798 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,798 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,798 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,798 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.1731809034116, worker=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,798 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-17T02:03:59,799 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=61 2024-11-17T02:03:59,799 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c6c0d7a0a7c0,45037,1731809032394, table=hbase:meta, region=1588230740 2024-11-17T02:03:59,799 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c6c0d7a0a7c0,45037,1731809032394, state=OPEN 2024-11-17T02:03:59,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,811 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,811 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,811 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,811 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731809069811 2024-11-17T02:03:59,812 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-17T02:03:59,811 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,813 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-17T02:03:59,814 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-17T02:03:59,814 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,814 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,814 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,45037,1731809032394 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,817 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9924be32211cddceef43e5b4866dc8, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,818 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a17ac9e270344fa2752f51f8196693b2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,818 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81f322eb57dcb42c3b4269a738bafd6d, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,818 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15c64fab21a3ca03ec0a7afabc19cdc9, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,818 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8356c0fdb856c376b8678f2c2597565a, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,818 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=112f5bbe15f72793585db9ba98950976, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,818 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7517bce9e04879764302cccd45af20fb, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,819 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fd33da09f9d938215ac0e7127ad70d5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,819 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03b7adb3afa9bebd86dc1da38125b982, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,819 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05534a4a027a1fdc90227b7a2e681933, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,819 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac10d67ff6c3f527cc0861e738032bdb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,819 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1056118c93716b24d18f749826cf316a, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,819 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663bac36df6c9873ad776bb3ad42409f, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,819 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74108f0b014075d4993f0eee396e39e5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,819 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=25a67f85380b4a3cd6890ab81d0b3a27, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,820 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e1c1fc49350743a9cde50c433c8cb19, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,820 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=899f31eacba6d2d0aedbee7d8bbb4c9d, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,820 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7934444caa228b9e5d6d7b28b59a796, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,820 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6eba00b85aa16d5ccbfc8e3c35ff298, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,820 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4748cc4c5ad3838a3d0568cee797c169, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,820 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77bd97fde6bd169f3e20553b45a65e2c, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,820 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f4fe90ec03839f5d26acd9d02569e56, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,820 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db157dde5b6770afbf869f7733d95efb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,821 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d86e1fe125f78846dcaa5b9921c72db, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,821 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eed07a820ab15d7af64462e2c1b70f7, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,821 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=037b7a93730957425bc54697b3594bb2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,821 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-17T02:03:59,821 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731809099821 2024-11-17T02:03:59,821 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731809159821 2024-11-17T02:03:59,821 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-17T02:03:59,821 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-17T02:03:59,821 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,821 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-17T02:03:59,821 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-17T02:03:59,822 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-17T02:03:59,822 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-17T02:03:59,822 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-17T02:03:59,822 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-17T02:03:59,823 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-17T02:03:59,825 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-17T02:03:59,825 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-17T02:03:59,825 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-17T02:03:59,825 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,825 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-17T02:03:59,825 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-17T02:03:59,826 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-17T02:03:59,827 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true 2024-11-17T02:03:59,827 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,827 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,827 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T02:03:59,827 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,827 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,827 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,827 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,827 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,827 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,45037,1731809032394 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,827 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-17T02:03:59,828 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=61 2024-11-17T02:03:59,828 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c6c0d7a0a7c0,45037,1731809032394, table=hbase:meta, region=1588230740 2024-11-17T02:03:59,828 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c6c0d7a0a7c0,45037,1731809032394, state=OPEN 2024-11-17T02:03:59,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,836 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,836 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,836 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,836 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731809069836 2024-11-17T02:03:59,837 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-17T02:03:59,837 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,838 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-17T02:03:59,839 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-17T02:03:59,839 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T02:03:59,839 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,840 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9924be32211cddceef43e5b4866dc8, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a17ac9e270344fa2752f51f8196693b2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81f322eb57dcb42c3b4269a738bafd6d, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15c64fab21a3ca03ec0a7afabc19cdc9, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8356c0fdb856c376b8678f2c2597565a, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=112f5bbe15f72793585db9ba98950976, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7517bce9e04879764302cccd45af20fb, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fd33da09f9d938215ac0e7127ad70d5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03b7adb3afa9bebd86dc1da38125b982, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05534a4a027a1fdc90227b7a2e681933, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac10d67ff6c3f527cc0861e738032bdb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1056118c93716b24d18f749826cf316a, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663bac36df6c9873ad776bb3ad42409f, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74108f0b014075d4993f0eee396e39e5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=25a67f85380b4a3cd6890ab81d0b3a27, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e1c1fc49350743a9cde50c433c8cb19, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=899f31eacba6d2d0aedbee7d8bbb4c9d, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7934444caa228b9e5d6d7b28b59a796, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6eba00b85aa16d5ccbfc8e3c35ff298, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4748cc4c5ad3838a3d0568cee797c169, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77bd97fde6bd169f3e20553b45a65e2c, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f4fe90ec03839f5d26acd9d02569e56, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db157dde5b6770afbf869f7733d95efb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d86e1fe125f78846dcaa5b9921c72db, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eed07a820ab15d7af64462e2c1b70f7, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=037b7a93730957425bc54697b3594bb2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,844 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-17T02:03:59,844 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731809099844 2024-11-17T02:03:59,844 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731809159844 2024-11-17T02:03:59,844 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-17T02:03:59,844 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-17T02:03:59,844 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,844 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-17T02:03:59,845 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-17T02:03:59,845 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-17T02:03:59,845 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-17T02:03:59,845 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-17T02:03:59,845 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-17T02:03:59,846 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-17T02:03:59,848 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-17T02:03:59,848 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-17T02:03:59,848 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-17T02:03:59,848 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-17T02:03:59,848 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-17T02:03:59,849 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-17T02:03:59,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true 2024-11-17T02:03:59,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T02:03:59,850 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,850 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,850 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,850 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,850 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-17T02:03:59,851 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=61 2024-11-17T02:03:59,851 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c6c0d7a0a7c0,45037,1731809032394, table=hbase:meta, region=1588230740 2024-11-17T02:03:59,852 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c6c0d7a0a7c0,45037,1731809032394, state=OPEN 2024-11-17T02:03:59,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,861 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,861 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,861 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,862 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731809069862 2024-11-17T02:03:59,862 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-17T02:03:59,862 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,863 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-17T02:03:59,864 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-17T02:03:59,864 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,864 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,864 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,867 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9924be32211cddceef43e5b4866dc8, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,867 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a17ac9e270344fa2752f51f8196693b2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,867 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81f322eb57dcb42c3b4269a738bafd6d, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,868 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15c64fab21a3ca03ec0a7afabc19cdc9, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,868 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8356c0fdb856c376b8678f2c2597565a, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,868 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=112f5bbe15f72793585db9ba98950976, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,868 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7517bce9e04879764302cccd45af20fb, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,868 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fd33da09f9d938215ac0e7127ad70d5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,868 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03b7adb3afa9bebd86dc1da38125b982, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,868 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05534a4a027a1fdc90227b7a2e681933, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac10d67ff6c3f527cc0861e738032bdb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1056118c93716b24d18f749826cf316a, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663bac36df6c9873ad776bb3ad42409f, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74108f0b014075d4993f0eee396e39e5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=25a67f85380b4a3cd6890ab81d0b3a27, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e1c1fc49350743a9cde50c433c8cb19, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=899f31eacba6d2d0aedbee7d8bbb4c9d, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7934444caa228b9e5d6d7b28b59a796, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6eba00b85aa16d5ccbfc8e3c35ff298, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4748cc4c5ad3838a3d0568cee797c169, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77bd97fde6bd169f3e20553b45a65e2c, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f4fe90ec03839f5d26acd9d02569e56, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db157dde5b6770afbf869f7733d95efb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d86e1fe125f78846dcaa5b9921c72db, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eed07a820ab15d7af64462e2c1b70f7, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=037b7a93730957425bc54697b3594bb2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,870 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-17T02:03:59,870 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731809099870 2024-11-17T02:03:59,870 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731809159870 2024-11-17T02:03:59,871 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-17T02:03:59,871 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-17T02:03:59,871 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,871 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-17T02:03:59,871 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-17T02:03:59,871 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-17T02:03:59,871 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-17T02:03:59,871 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-17T02:03:59,871 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-17T02:03:59,872 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-17T02:03:59,874 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-17T02:03:59,874 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-17T02:03:59,874 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-17T02:03:59,874 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-17T02:03:59,875 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-17T02:03:59,876 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true 2024-11-17T02:03:59,876 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,876 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,876 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,876 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,876 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,876 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-17T02:03:59,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=61 2024-11-17T02:03:59,877 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c6c0d7a0a7c0,45037,1731809032394, table=hbase:meta, region=1588230740 2024-11-17T02:03:59,877 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c6c0d7a0a7c0,45037,1731809032394, state=OPEN 2024-11-17T02:03:59,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,886 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,886 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,886 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,886 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731809069886 2024-11-17T02:03:59,886 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-17T02:03:59,886 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,887 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-17T02:03:59,888 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-17T02:03:59,888 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,888 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,888 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,890 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9924be32211cddceef43e5b4866dc8, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,891 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a17ac9e270344fa2752f51f8196693b2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,891 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81f322eb57dcb42c3b4269a738bafd6d, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,891 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15c64fab21a3ca03ec0a7afabc19cdc9, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,891 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8356c0fdb856c376b8678f2c2597565a, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,891 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=112f5bbe15f72793585db9ba98950976, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,891 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7517bce9e04879764302cccd45af20fb, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,891 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fd33da09f9d938215ac0e7127ad70d5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,891 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03b7adb3afa9bebd86dc1da38125b982, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,892 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05534a4a027a1fdc90227b7a2e681933, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,892 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac10d67ff6c3f527cc0861e738032bdb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,892 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1056118c93716b24d18f749826cf316a, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,892 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663bac36df6c9873ad776bb3ad42409f, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,892 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74108f0b014075d4993f0eee396e39e5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,892 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=25a67f85380b4a3cd6890ab81d0b3a27, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,892 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e1c1fc49350743a9cde50c433c8cb19, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,892 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=899f31eacba6d2d0aedbee7d8bbb4c9d, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,892 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7934444caa228b9e5d6d7b28b59a796, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,893 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6eba00b85aa16d5ccbfc8e3c35ff298, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,893 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4748cc4c5ad3838a3d0568cee797c169, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,893 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77bd97fde6bd169f3e20553b45a65e2c, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,893 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f4fe90ec03839f5d26acd9d02569e56, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,893 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db157dde5b6770afbf869f7733d95efb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,893 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d86e1fe125f78846dcaa5b9921c72db, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,893 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eed07a820ab15d7af64462e2c1b70f7, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,893 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=037b7a93730957425bc54697b3594bb2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,893 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-17T02:03:59,893 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731809099893 2024-11-17T02:03:59,893 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731809159893 2024-11-17T02:03:59,894 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-17T02:03:59,894 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-17T02:03:59,894 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,894 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-17T02:03:59,894 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-17T02:03:59,894 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-17T02:03:59,894 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-17T02:03:59,894 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-17T02:03:59,895 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-17T02:03:59,895 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-17T02:03:59,897 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-17T02:03:59,897 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-17T02:03:59,897 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-17T02:03:59,898 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-17T02:03:59,904 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true 2024-11-17T02:03:59,905 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,905 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,905 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure c6c0d7a0a7c0%2C34033%2C1731809032455.meta.1731809034432.meta, worker=c6c0d7a0a7c0,33127,1731809032263 didn't hold the lock before restarting, skip acquiring lock. 2024-11-17T02:03:59,905 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-17T02:03:59,905 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32803 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=61 2024-11-17T02:03:59,906 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c6c0d7a0a7c0,45037,1731809032394, table=hbase:meta, region=1588230740 2024-11-17T02:03:59,906 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c6c0d7a0a7c0,45037,1731809032394, state=OPEN 2024-11-17T02:03:59,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,917 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,917 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,918 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,918 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731809069918 2024-11-17T02:03:59,918 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-17T02:03:59,918 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,919 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-17T02:03:59,920 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-17T02:03:59,920 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true 2024-11-17T02:03:59,920 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,920 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,924 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9924be32211cddceef43e5b4866dc8, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,924 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a17ac9e270344fa2752f51f8196693b2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,924 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81f322eb57dcb42c3b4269a738bafd6d, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,924 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15c64fab21a3ca03ec0a7afabc19cdc9, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,924 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8356c0fdb856c376b8678f2c2597565a, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,924 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=112f5bbe15f72793585db9ba98950976, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,924 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7517bce9e04879764302cccd45af20fb, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,925 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fd33da09f9d938215ac0e7127ad70d5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,925 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03b7adb3afa9bebd86dc1da38125b982, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,925 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05534a4a027a1fdc90227b7a2e681933, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,925 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac10d67ff6c3f527cc0861e738032bdb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,925 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1056118c93716b24d18f749826cf316a, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,925 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663bac36df6c9873ad776bb3ad42409f, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,925 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74108f0b014075d4993f0eee396e39e5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,925 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=25a67f85380b4a3cd6890ab81d0b3a27, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,925 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e1c1fc49350743a9cde50c433c8cb19, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=899f31eacba6d2d0aedbee7d8bbb4c9d, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7934444caa228b9e5d6d7b28b59a796, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6eba00b85aa16d5ccbfc8e3c35ff298, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4748cc4c5ad3838a3d0568cee797c169, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77bd97fde6bd169f3e20553b45a65e2c, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f4fe90ec03839f5d26acd9d02569e56, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db157dde5b6770afbf869f7733d95efb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d86e1fe125f78846dcaa5b9921c72db, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eed07a820ab15d7af64462e2c1b70f7, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=037b7a93730957425bc54697b3594bb2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,926 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-17T02:03:59,926 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731809099926 2024-11-17T02:03:59,926 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731809159926 2024-11-17T02:03:59,927 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-17T02:03:59,927 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-17T02:03:59,927 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,927 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-17T02:03:59,927 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-17T02:03:59,927 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-17T02:03:59,927 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-17T02:03:59,927 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-17T02:03:59,927 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-17T02:03:59,928 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-17T02:03:59,929 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-17T02:03:59,929 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-17T02:03:59,930 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-17T02:03:59,930 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true 2024-11-17T02:03:59,930 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-17T02:03:59,930 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-17T02:03:59,931 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=61 2024-11-17T02:03:59,931 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=c6c0d7a0a7c0,45037,1731809032394, table=hbase:meta, region=1588230740 2024-11-17T02:03:59,931 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c6c0d7a0a7c0,45037,1731809032394, state=OPEN 2024-11-17T02:03:59,942 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,942 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,942 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T02:03:59,942 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,942 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,942 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T02:03:59,943 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731809069943 2024-11-17T02:03:59,943 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-17T02:03:59,943 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T02:03:59,944 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-17T02:03:59,945 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=57, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=65, state=RUNNABLE, hasLock=true; OpenRegionProcedure 03b7adb3afa9bebd86dc1da38125b982, server=c6c0d7a0a7c0,33127,1731809032263:java.lang.RuntimeException: inject code bug; ServerCrashProcedure c6c0d7a0a7c0,34033,1731809032455, splitWal=true, meta=true exec-time=2.5650 sec 2024-11-17T02:03:59,948 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9924be32211cddceef43e5b4866dc8, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,948 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a17ac9e270344fa2752f51f8196693b2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,949 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=81f322eb57dcb42c3b4269a738bafd6d, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,949 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=15c64fab21a3ca03ec0a7afabc19cdc9, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,949 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8356c0fdb856c376b8678f2c2597565a, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,949 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=112f5bbe15f72793585db9ba98950976, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,949 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7517bce9e04879764302cccd45af20fb, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,949 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6fd33da09f9d938215ac0e7127ad70d5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,949 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03b7adb3afa9bebd86dc1da38125b982, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,950 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=05534a4a027a1fdc90227b7a2e681933, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,950 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac10d67ff6c3f527cc0861e738032bdb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,950 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1056118c93716b24d18f749826cf316a, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,950 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=663bac36df6c9873ad776bb3ad42409f, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,950 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=74108f0b014075d4993f0eee396e39e5, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,950 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=25a67f85380b4a3cd6890ab81d0b3a27, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,951 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e1c1fc49350743a9cde50c433c8cb19, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,951 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=899f31eacba6d2d0aedbee7d8bbb4c9d, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,951 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7934444caa228b9e5d6d7b28b59a796, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,951 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6eba00b85aa16d5ccbfc8e3c35ff298, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,951 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4748cc4c5ad3838a3d0568cee797c169, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,951 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=77bd97fde6bd169f3e20553b45a65e2c, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,951 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f4fe90ec03839f5d26acd9d02569e56, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db157dde5b6770afbf869f7733d95efb, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d86e1fe125f78846dcaa5b9921c72db, regionState=OPEN, lastHost=c6c0d7a0a7c0,33127,1731809032263, regionLocation=c6c0d7a0a7c0,33127,1731809032263, openSeqNum=2 2024-11-17T02:03:59,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eed07a820ab15d7af64462e2c1b70f7, regionState=OPENING, lastHost=c6c0d7a0a7c0,34033,1731809032455, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=037b7a93730957425bc54697b3594bb2, regionState=OPEN, lastHost=c6c0d7a0a7c0,45037,1731809032394, regionLocation=c6c0d7a0a7c0,45037,1731809032394, openSeqNum=2 2024-11-17T02:03:59,952 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-17T02:03:59,952 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731809099952 2024-11-17T02:03:59,952 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731809159952 2024-11-17T02:03:59,952 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-17T02:03:59,966 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=350 (was 384), OpenFileDescriptor=628 (was 649), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=129 (was 122) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5738 (was 5775) 2024-11-17T02:03:59,966 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-17T02:03:59,966 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T02:03:59,966 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T02:03:59,966 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T02:03:59,967 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T02:03:59,967 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T02:03:59,967 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-17T02:03:59,967 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1546128933, stopped=false 2024-11-17T02:03:59,967 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c6c0d7a0a7c0,32803,1731809031530 2024-11-17T02:03:59,974 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T02:03:59,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-17T02:03:59,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-17T02:03:59,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-17T02:03:59,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T02:03:59,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T02:03:59,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T02:03:59,977 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:03:59,978 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T02:03:59,978 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T02:03:59,978 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T02:03:59,978 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T02:03:59,978 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T02:03:59,978 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T02:03:59,979 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c6c0d7a0a7c0,33127,1731809032263' ***** 2024-11-17T02:03:59,979 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-17T02:03:59,979 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c6c0d7a0a7c0,45037,1731809032394' ***** 2024-11-17T02:03:59,979 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-17T02:03:59,980 INFO [RS:0;c6c0d7a0a7c0:33127 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-17T02:03:59,980 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-17T02:03:59,980 INFO [RS:0;c6c0d7a0a7c0:33127 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-17T02:03:59,980 INFO [RS:1;c6c0d7a0a7c0:45037 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-17T02:03:59,980 INFO [RS:0;c6c0d7a0a7c0:33127 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-17T02:03:59,980 INFO [RS:1;c6c0d7a0a7c0:45037 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-17T02:03:59,980 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-17T02:03:59,980 INFO [RS:1;c6c0d7a0a7c0:45037 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-17T02:03:59,980 INFO [RS:0;c6c0d7a0a7c0:33127 {}] regionserver.HRegionServer(3091): Received CLOSE for 03b7adb3afa9bebd86dc1da38125b982 2024-11-17T02:03:59,980 INFO [RS:1;c6c0d7a0a7c0:45037 {}] regionserver.HRegionServer(3091): Received CLOSE for ac10d67ff6c3f527cc0861e738032bdb 2024-11-17T02:03:59,981 INFO [RS:0;c6c0d7a0a7c0:33127 {}] regionserver.HRegionServer(3091): Received CLOSE for 8356c0fdb856c376b8678f2c2597565a 2024-11-17T02:03:59,981 INFO [RS:0;c6c0d7a0a7c0:33127 {}] regionserver.HRegionServer(3091): Received CLOSE for 25a67f85380b4a3cd6890ab81d0b3a27 2024-11-17T02:03:59,981 INFO [RS:1;c6c0d7a0a7c0:45037 {}] regionserver.HRegionServer(3091): Received CLOSE for db157dde5b6770afbf869f7733d95efb 2024-11-17T02:03:59,981 INFO [RS:0;c6c0d7a0a7c0:33127 {}] regionserver.HRegionServer(3091): Received CLOSE for 8e1c1fc49350743a9cde50c433c8cb19 2024-11-17T02:03:59,981 INFO [RS:1;c6c0d7a0a7c0:45037 {}] regionserver.HRegionServer(3091): Received CLOSE for 899f31eacba6d2d0aedbee7d8bbb4c9d 2024-11-17T02:03:59,981 INFO [RS:0;c6c0d7a0a7c0:33127 {}] regionserver.HRegionServer(3091): Received CLOSE for 663bac36df6c9873ad776bb3ad42409f 2024-11-17T02:03:59,981 INFO [RS:1;c6c0d7a0a7c0:45037 {}] regionserver.HRegionServer(3091): Received CLOSE for 7eed07a820ab15d7af64462e2c1b70f7 2024-11-17T02:03:59,981 INFO [RS:0;c6c0d7a0a7c0:33127 {}] regionserver.HRegionServer(3091): Received CLOSE for 7517bce9e04879764302cccd45af20fb 2024-11-17T02:03:59,981 INFO [RS:1;c6c0d7a0a7c0:45037 {}] regionserver.HRegionServer(3091): Received CLOSE for b6eba00b85aa16d5ccbfc8e3c35ff298 2024-11-17T02:03:59,981 INFO [RS:0;c6c0d7a0a7c0:33127 {}] regionserver.HRegionServer(3091): Received CLOSE for 81f322eb57dcb42c3b4269a738bafd6d 2024-11-17T02:03:59,981 INFO [RS:1;c6c0d7a0a7c0:45037 {}] regionserver.HRegionServer(3091): Received CLOSE for 4748cc4c5ad3838a3d0568cee797c169 2024-11-17T02:03:59,981 INFO [RS:0;c6c0d7a0a7c0:33127 {}] regionserver.HRegionServer(3091): Received CLOSE for 9d86e1fe125f78846dcaa5b9921c72db 2024-11-17T02:03:59,981 INFO [RS:1;c6c0d7a0a7c0:45037 {}] regionserver.HRegionServer(3091): Received CLOSE for 6fd33da09f9d938215ac0e7127ad70d5 2024-11-17T02:03:59,981 INFO [RS:0;c6c0d7a0a7c0:33127 {}] regionserver.HRegionServer(3091): Received CLOSE for 77bd97fde6bd169f3e20553b45a65e2c 2024-11-17T02:03:59,981 INFO [RS:1;c6c0d7a0a7c0:45037 {}] regionserver.HRegionServer(3091): Received CLOSE for 15c64fab21a3ca03ec0a7afabc19cdc9 2024-11-17T02:03:59,981 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 03b7adb3afa9bebd86dc1da38125b982, disabling compactions & flushes 2024-11-17T02:03:59,981 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ac10d67ff6c3f527cc0861e738032bdb, disabling compactions & flushes 2024-11-17T02:03:59,981 INFO [RS:0;c6c0d7a0a7c0:33127 {}] regionserver.HRegionServer(3091): Received CLOSE for 05534a4a027a1fdc90227b7a2e681933 2024-11-17T02:03:59,981 INFO [RS:1;c6c0d7a0a7c0:45037 {}] regionserver.HRegionServer(3091): Received CLOSE for 112f5bbe15f72793585db9ba98950976 2024-11-17T02:03:59,981 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1731809034900.03b7adb3afa9bebd86dc1da38125b982. 2024-11-17T02:03:59,981 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1731809034900.ac10d67ff6c3f527cc0861e738032bdb. 2024-11-17T02:03:59,981 INFO [RS:0;c6c0d7a0a7c0:33127 {}] regionserver.HRegionServer(3091): Received CLOSE for de9924be32211cddceef43e5b4866dc8 2024-11-17T02:03:59,982 INFO [RS:1;c6c0d7a0a7c0:45037 {}] regionserver.HRegionServer(3091): Received CLOSE for 1056118c93716b24d18f749826cf316a 2024-11-17T02:03:59,982 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1731809034900.03b7adb3afa9bebd86dc1da38125b982. 2024-11-17T02:03:59,982 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1731809034900.ac10d67ff6c3f527cc0861e738032bdb. 2024-11-17T02:03:59,982 INFO [RS:0;c6c0d7a0a7c0:33127 {}] regionserver.HRegionServer(3091): Received CLOSE for 2f4fe90ec03839f5d26acd9d02569e56 2024-11-17T02:03:59,982 INFO [RS:1;c6c0d7a0a7c0:45037 {}] regionserver.HRegionServer(3091): Received CLOSE for a17ac9e270344fa2752f51f8196693b2 2024-11-17T02:03:59,982 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1731809034900.03b7adb3afa9bebd86dc1da38125b982. after waiting 0 ms 2024-11-17T02:03:59,982 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1731809034900.ac10d67ff6c3f527cc0861e738032bdb. after waiting 0 ms 2024-11-17T02:03:59,982 INFO [RS:0;c6c0d7a0a7c0:33127 {}] regionserver.HRegionServer(3091): Received CLOSE for b7934444caa228b9e5d6d7b28b59a796 2024-11-17T02:03:59,982 INFO [RS:1;c6c0d7a0a7c0:45037 {}] regionserver.HRegionServer(3091): Received CLOSE for 74108f0b014075d4993f0eee396e39e5 2024-11-17T02:03:59,982 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1731809034900.03b7adb3afa9bebd86dc1da38125b982. 2024-11-17T02:03:59,982 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1731809034900.ac10d67ff6c3f527cc0861e738032bdb. 2024-11-17T02:03:59,982 INFO [RS:1;c6c0d7a0a7c0:45037 {}] regionserver.HRegionServer(3091): Received CLOSE for 037b7a93730957425bc54697b3594bb2 2024-11-17T02:03:59,982 INFO [RS:0;c6c0d7a0a7c0:33127 {}] regionserver.HRegionServer(959): stopping server c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:03:59,982 INFO [RS:1;c6c0d7a0a7c0:45037 {}] regionserver.HRegionServer(959): stopping server c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:03:59,982 INFO [RS:0;c6c0d7a0a7c0:33127 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-17T02:03:59,982 INFO [RS:1;c6c0d7a0a7c0:45037 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-17T02:03:59,982 INFO [RS:1;c6c0d7a0a7c0:45037 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;c6c0d7a0a7c0:45037. 2024-11-17T02:03:59,982 INFO [RS:0;c6c0d7a0a7c0:33127 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c6c0d7a0a7c0:33127. 2024-11-17T02:03:59,982 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T02:03:59,982 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T02:03:59,982 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T02:03:59,982 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T02:03:59,983 INFO [RS:0;c6c0d7a0a7c0:33127 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-11-17T02:03:59,983 INFO [RS:1;c6c0d7a0a7c0:45037 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-17T02:03:59,983 INFO [RS:1;c6c0d7a0a7c0:45037 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-17T02:03:59,983 INFO [RS:1;c6c0d7a0a7c0:45037 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-17T02:03:59,983 INFO [RS:1;c6c0d7a0a7c0:45037 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-17T02:03:59,983 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] regionserver.HRegionServer(1351): Waiting on 03b7adb3afa9bebd86dc1da38125b982, 05534a4a027a1fdc90227b7a2e681933, 25a67f85380b4a3cd6890ab81d0b3a27, 2f4fe90ec03839f5d26acd9d02569e56, 663bac36df6c9873ad776bb3ad42409f, 7517bce9e04879764302cccd45af20fb, 77bd97fde6bd169f3e20553b45a65e2c, 81f322eb57dcb42c3b4269a738bafd6d, 8356c0fdb856c376b8678f2c2597565a, 8e1c1fc49350743a9cde50c433c8cb19, 9d86e1fe125f78846dcaa5b9921c72db, b7934444caa228b9e5d6d7b28b59a796, de9924be32211cddceef43e5b4866dc8 2024-11-17T02:03:59,987 INFO [RS:1;c6c0d7a0a7c0:45037 {}] regionserver.HRegionServer(1321): Waiting on 14 regions to close 2024-11-17T02:03:59,987 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] regionserver.HRegionServer(1351): Waiting on 037b7a93730957425bc54697b3594bb2, 1056118c93716b24d18f749826cf316a, 112f5bbe15f72793585db9ba98950976, 1588230740, 15c64fab21a3ca03ec0a7afabc19cdc9, 4748cc4c5ad3838a3d0568cee797c169, 6fd33da09f9d938215ac0e7127ad70d5, 74108f0b014075d4993f0eee396e39e5, 7eed07a820ab15d7af64462e2c1b70f7, 899f31eacba6d2d0aedbee7d8bbb4c9d, a17ac9e270344fa2752f51f8196693b2, ac10d67ff6c3f527cc0861e738032bdb, b6eba00b85aa16d5ccbfc8e3c35ff298, db157dde5b6770afbf869f7733d95efb 2024-11-17T02:03:59,987 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-17T02:03:59,987 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-17T02:03:59,987 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-17T02:03:59,987 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-17T02:03:59,987 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-17T02:03:59,987 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.64 KB heapSize=5.42 KB 2024-11-17T02:03:59,991 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/03b7adb3afa9bebd86dc1da38125b982/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T02:03:59,993 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/ac10d67ff6c3f527cc0861e738032bdb/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T02:03:59,994 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1731809034900.03b7adb3afa9bebd86dc1da38125b982. 2024-11-17T02:03:59,994 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 03b7adb3afa9bebd86dc1da38125b982: Waiting for close lock at 1731809039981Running coprocessor pre-close hooks at 1731809039981Disabling compacts and flushes for region at 1731809039981Disabling writes for close at 1731809039982 (+1 ms)Writing region close event to WAL at 1731809039983 (+1 ms)Running coprocessor post-close hooks at 1731809039992 (+9 ms)Closed at 1731809039994 (+2 ms) 2024-11-17T02:03:59,995 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1731809034900.03b7adb3afa9bebd86dc1da38125b982. 2024-11-17T02:03:59,995 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8356c0fdb856c376b8678f2c2597565a, disabling compactions & flushes 2024-11-17T02:03:59,995 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1731809034900.8356c0fdb856c376b8678f2c2597565a. 2024-11-17T02:03:59,995 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1731809034900.8356c0fdb856c376b8678f2c2597565a. 2024-11-17T02:03:59,995 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1731809034900.8356c0fdb856c376b8678f2c2597565a. after waiting 0 ms 2024-11-17T02:03:59,995 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1731809034900.8356c0fdb856c376b8678f2c2597565a. 2024-11-17T02:03:59,998 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1731809034900.ac10d67ff6c3f527cc0861e738032bdb. 2024-11-17T02:03:59,999 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ac10d67ff6c3f527cc0861e738032bdb: Waiting for close lock at 1731809039981Running coprocessor pre-close hooks at 1731809039981Disabling compacts and flushes for region at 1731809039981Disabling writes for close at 1731809039982 (+1 ms)Writing region close event to WAL at 1731809039987 (+5 ms)Running coprocessor post-close hooks at 1731809039998 (+11 ms)Closed at 1731809039998 2024-11-17T02:03:59,999 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1731809034900.ac10d67ff6c3f527cc0861e738032bdb. 2024-11-17T02:03:59,999 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing db157dde5b6770afbf869f7733d95efb, disabling compactions & flushes 2024-11-17T02:03:59,999 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1731809034900.db157dde5b6770afbf869f7733d95efb. 2024-11-17T02:03:59,999 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1731809034900.db157dde5b6770afbf869f7733d95efb. 2024-11-17T02:03:59,999 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1731809034900.db157dde5b6770afbf869f7733d95efb. after waiting 0 ms 2024-11-17T02:03:59,999 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1731809034900.db157dde5b6770afbf869f7733d95efb. 2024-11-17T02:04:00,004 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/db157dde5b6770afbf869f7733d95efb/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T02:04:00,005 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1731809034900.db157dde5b6770afbf869f7733d95efb. 2024-11-17T02:04:00,005 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for db157dde5b6770afbf869f7733d95efb: Waiting for close lock at 1731809039999Running coprocessor pre-close hooks at 1731809039999Disabling compacts and flushes for region at 1731809039999Disabling writes for close at 1731809039999Writing region close event to WAL at 1731809040000 (+1 ms)Running coprocessor post-close hooks at 1731809040005 (+5 ms)Closed at 1731809040005 2024-11-17T02:04:00,005 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1731809034900.db157dde5b6770afbf869f7733d95efb. 2024-11-17T02:04:00,005 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 899f31eacba6d2d0aedbee7d8bbb4c9d, disabling compactions & flushes 2024-11-17T02:04:00,005 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1731809034900.899f31eacba6d2d0aedbee7d8bbb4c9d. 2024-11-17T02:04:00,005 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1731809034900.899f31eacba6d2d0aedbee7d8bbb4c9d. 2024-11-17T02:04:00,005 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1731809034900.899f31eacba6d2d0aedbee7d8bbb4c9d. after waiting 0 ms 2024-11-17T02:04:00,005 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1731809034900.899f31eacba6d2d0aedbee7d8bbb4c9d. 2024-11-17T02:04:00,008 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/hbase/meta/1588230740/.tmp/info/45d0f8b570074839885917546373acf9 is 138, key is test,fff,1731809034900.112f5bbe15f72793585db9ba98950976./info:regioninfo/1731809038779/Put/seqid=0 2024-11-17T02:04:00,016 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/8356c0fdb856c376b8678f2c2597565a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T02:04:00,017 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1731809034900.8356c0fdb856c376b8678f2c2597565a. 2024-11-17T02:04:00,017 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8356c0fdb856c376b8678f2c2597565a: Waiting for close lock at 1731809039995Running coprocessor pre-close hooks at 1731809039995Disabling compacts and flushes for region at 1731809039995Disabling writes for close at 1731809039995Writing region close event to WAL at 1731809039996 (+1 ms)Running coprocessor post-close hooks at 1731809040017 (+21 ms)Closed at 1731809040017 2024-11-17T02:04:00,017 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1731809034900.8356c0fdb856c376b8678f2c2597565a. 2024-11-17T02:04:00,017 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 25a67f85380b4a3cd6890ab81d0b3a27, disabling compactions & flushes 2024-11-17T02:04:00,018 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1731809034900.25a67f85380b4a3cd6890ab81d0b3a27. 2024-11-17T02:04:00,018 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1731809034900.25a67f85380b4a3cd6890ab81d0b3a27. 2024-11-17T02:04:00,018 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1731809034900.25a67f85380b4a3cd6890ab81d0b3a27. after waiting 0 ms 2024-11-17T02:04:00,018 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1731809034900.25a67f85380b4a3cd6890ab81d0b3a27. 2024-11-17T02:04:00,021 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/899f31eacba6d2d0aedbee7d8bbb4c9d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T02:04:00,022 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1731809034900.899f31eacba6d2d0aedbee7d8bbb4c9d. 2024-11-17T02:04:00,022 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 899f31eacba6d2d0aedbee7d8bbb4c9d: Waiting for close lock at 1731809040005Running coprocessor pre-close hooks at 1731809040005Disabling compacts and flushes for region at 1731809040005Disabling writes for close at 1731809040005Writing region close event to WAL at 1731809040015 (+10 ms)Running coprocessor post-close hooks at 1731809040022 (+7 ms)Closed at 1731809040022 2024-11-17T02:04:00,022 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1731809034900.899f31eacba6d2d0aedbee7d8bbb4c9d. 2024-11-17T02:04:00,022 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7eed07a820ab15d7af64462e2c1b70f7, disabling compactions & flushes 2024-11-17T02:04:00,022 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1731809034900.7eed07a820ab15d7af64462e2c1b70f7. 2024-11-17T02:04:00,022 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1731809034900.7eed07a820ab15d7af64462e2c1b70f7. 2024-11-17T02:04:00,022 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1731809034900.7eed07a820ab15d7af64462e2c1b70f7. after waiting 0 ms 2024-11-17T02:04:00,022 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1731809034900.7eed07a820ab15d7af64462e2c1b70f7. 2024-11-17T02:04:00,029 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/25a67f85380b4a3cd6890ab81d0b3a27/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T02:04:00,030 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1731809034900.25a67f85380b4a3cd6890ab81d0b3a27. 2024-11-17T02:04:00,031 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 25a67f85380b4a3cd6890ab81d0b3a27: Waiting for close lock at 1731809040017Running coprocessor pre-close hooks at 1731809040017Disabling compacts and flushes for region at 1731809040017Disabling writes for close at 1731809040018 (+1 ms)Writing region close event to WAL at 1731809040018Running coprocessor post-close hooks at 1731809040030 (+12 ms)Closed at 1731809040030 2024-11-17T02:04:00,031 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1731809034900.25a67f85380b4a3cd6890ab81d0b3a27. 2024-11-17T02:04:00,031 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8e1c1fc49350743a9cde50c433c8cb19, disabling compactions & flushes 2024-11-17T02:04:00,031 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ppp,1731809034900.8e1c1fc49350743a9cde50c433c8cb19. 2024-11-17T02:04:00,031 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1731809034900.8e1c1fc49350743a9cde50c433c8cb19. 2024-11-17T02:04:00,031 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1731809034900.8e1c1fc49350743a9cde50c433c8cb19. after waiting 0 ms 2024-11-17T02:04:00,031 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1731809034900.8e1c1fc49350743a9cde50c433c8cb19. 2024-11-17T02:04:00,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741869_1045 (size=8177) 2024-11-17T02:04:00,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741869_1045 (size=8177) 2024-11-17T02:04:00,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741869_1045 (size=8177) 2024-11-17T02:04:00,035 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.64 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/hbase/meta/1588230740/.tmp/info/45d0f8b570074839885917546373acf9 2024-11-17T02:04:00,039 INFO [regionserver/c6c0d7a0a7c0:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-17T02:04:00,039 INFO [regionserver/c6c0d7a0a7c0:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-17T02:04:00,043 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/8e1c1fc49350743a9cde50c433c8cb19/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T02:04:00,044 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ppp,1731809034900.8e1c1fc49350743a9cde50c433c8cb19. 2024-11-17T02:04:00,044 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8e1c1fc49350743a9cde50c433c8cb19: Waiting for close lock at 1731809040031Running coprocessor pre-close hooks at 1731809040031Disabling compacts and flushes for region at 1731809040031Disabling writes for close at 1731809040031Writing region close event to WAL at 1731809040035 (+4 ms)Running coprocessor post-close hooks at 1731809040044 (+9 ms)Closed at 1731809040044 2024-11-17T02:04:00,044 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ppp,1731809034900.8e1c1fc49350743a9cde50c433c8cb19. 2024-11-17T02:04:00,045 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 663bac36df6c9873ad776bb3ad42409f, disabling compactions & flushes 2024-11-17T02:04:00,045 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1731809034900.663bac36df6c9873ad776bb3ad42409f. 2024-11-17T02:04:00,045 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1731809034900.663bac36df6c9873ad776bb3ad42409f. 2024-11-17T02:04:00,045 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1731809034900.663bac36df6c9873ad776bb3ad42409f. after waiting 0 ms 2024-11-17T02:04:00,045 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1731809034900.663bac36df6c9873ad776bb3ad42409f. 2024-11-17T02:04:00,045 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-11-17T02:04:00,046 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/hbase/meta/1588230740/.tmp/info/45d0f8b570074839885917546373acf9 as hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/hbase/meta/1588230740/info/45d0f8b570074839885917546373acf9 2024-11-17T02:04:00,049 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/7eed07a820ab15d7af64462e2c1b70f7/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T02:04:00,050 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1731809034900.7eed07a820ab15d7af64462e2c1b70f7. 2024-11-17T02:04:00,051 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7eed07a820ab15d7af64462e2c1b70f7: Waiting for close lock at 1731809040022Running coprocessor pre-close hooks at 1731809040022Disabling compacts and flushes for region at 1731809040022Disabling writes for close at 1731809040022Writing region close event to WAL at 1731809040029 (+7 ms)Running coprocessor post-close hooks at 1731809040050 (+21 ms)Closed at 1731809040050 2024-11-17T02:04:00,051 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1731809034900.7eed07a820ab15d7af64462e2c1b70f7. 2024-11-17T02:04:00,051 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b6eba00b85aa16d5ccbfc8e3c35ff298, disabling compactions & flushes 2024-11-17T02:04:00,051 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1731809034900.b6eba00b85aa16d5ccbfc8e3c35ff298. 2024-11-17T02:04:00,051 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1731809034900.b6eba00b85aa16d5ccbfc8e3c35ff298. 2024-11-17T02:04:00,051 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1731809034900.b6eba00b85aa16d5ccbfc8e3c35ff298. after waiting 0 ms 2024-11-17T02:04:00,051 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1731809034900.b6eba00b85aa16d5ccbfc8e3c35ff298. 2024-11-17T02:04:00,056 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/663bac36df6c9873ad776bb3ad42409f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T02:04:00,056 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/hbase/meta/1588230740/info/45d0f8b570074839885917546373acf9, entries=24, sequenceid=71, filesize=8.0 K 2024-11-17T02:04:00,056 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1731809034900.663bac36df6c9873ad776bb3ad42409f. 2024-11-17T02:04:00,057 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 663bac36df6c9873ad776bb3ad42409f: Waiting for close lock at 1731809040045Running coprocessor pre-close hooks at 1731809040045Disabling compacts and flushes for region at 1731809040045Disabling writes for close at 1731809040045Writing region close event to WAL at 1731809040049 (+4 ms)Running coprocessor post-close hooks at 1731809040056 (+7 ms)Closed at 1731809040056 2024-11-17T02:04:00,057 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1731809034900.663bac36df6c9873ad776bb3ad42409f. 2024-11-17T02:04:00,057 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7517bce9e04879764302cccd45af20fb, disabling compactions & flushes 2024-11-17T02:04:00,057 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1731809034900.7517bce9e04879764302cccd45af20fb. 2024-11-17T02:04:00,057 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1731809034900.7517bce9e04879764302cccd45af20fb. 2024-11-17T02:04:00,057 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1731809034900.7517bce9e04879764302cccd45af20fb. after waiting 0 ms 2024-11-17T02:04:00,057 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1731809034900.7517bce9e04879764302cccd45af20fb. 2024-11-17T02:04:00,057 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.64 KB/2708, heapSize ~4.66 KB/4768, currentSize=0 B/0 for 1588230740 in 70ms, sequenceid=71, compaction requested=false 2024-11-17T02:04:00,058 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/b6eba00b85aa16d5ccbfc8e3c35ff298/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T02:04:00,059 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1731809034900.b6eba00b85aa16d5ccbfc8e3c35ff298. 2024-11-17T02:04:00,059 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b6eba00b85aa16d5ccbfc8e3c35ff298: Waiting for close lock at 1731809040051Running coprocessor pre-close hooks at 1731809040051Disabling compacts and flushes for region at 1731809040051Disabling writes for close at 1731809040051Writing region close event to WAL at 1731809040051Running coprocessor post-close hooks at 1731809040059 (+8 ms)Closed at 1731809040059 2024-11-17T02:04:00,059 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1731809034900.b6eba00b85aa16d5ccbfc8e3c35ff298. 2024-11-17T02:04:00,059 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4748cc4c5ad3838a3d0568cee797c169, disabling compactions & flushes 2024-11-17T02:04:00,059 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1731809034900.4748cc4c5ad3838a3d0568cee797c169. 2024-11-17T02:04:00,059 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1731809034900.4748cc4c5ad3838a3d0568cee797c169. 2024-11-17T02:04:00,059 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1731809034900.4748cc4c5ad3838a3d0568cee797c169. after waiting 0 ms 2024-11-17T02:04:00,059 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1731809034900.4748cc4c5ad3838a3d0568cee797c169. 2024-11-17T02:04:00,063 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/7517bce9e04879764302cccd45af20fb/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T02:04:00,064 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1731809034900.7517bce9e04879764302cccd45af20fb. 2024-11-17T02:04:00,064 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7517bce9e04879764302cccd45af20fb: Waiting for close lock at 1731809040057Running coprocessor pre-close hooks at 1731809040057Disabling compacts and flushes for region at 1731809040057Disabling writes for close at 1731809040057Writing region close event to WAL at 1731809040057Running coprocessor post-close hooks at 1731809040064 (+7 ms)Closed at 1731809040064 2024-11-17T02:04:00,064 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1731809034900.7517bce9e04879764302cccd45af20fb. 2024-11-17T02:04:00,064 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 81f322eb57dcb42c3b4269a738bafd6d, disabling compactions & flushes 2024-11-17T02:04:00,064 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1731809034900.81f322eb57dcb42c3b4269a738bafd6d. 2024-11-17T02:04:00,065 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1731809034900.81f322eb57dcb42c3b4269a738bafd6d. 2024-11-17T02:04:00,065 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1731809034900.81f322eb57dcb42c3b4269a738bafd6d. after waiting 0 ms 2024-11-17T02:04:00,065 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1731809034900.81f322eb57dcb42c3b4269a738bafd6d. 2024-11-17T02:04:00,067 INFO [regionserver/c6c0d7a0a7c0:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T02:04:00,067 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/4748cc4c5ad3838a3d0568cee797c169/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T02:04:00,067 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/hbase/meta/1588230740/recovered.edits/74.seqid, newMaxSeqId=74, maxSeqId=60 2024-11-17T02:04:00,067 INFO [regionserver/c6c0d7a0a7c0:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T02:04:00,067 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1731809034900.4748cc4c5ad3838a3d0568cee797c169. 2024-11-17T02:04:00,068 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4748cc4c5ad3838a3d0568cee797c169: Waiting for close lock at 1731809040059Running coprocessor pre-close hooks at 1731809040059Disabling compacts and flushes for region at 1731809040059Disabling writes for close at 1731809040059Writing region close event to WAL at 1731809040060 (+1 ms)Running coprocessor post-close hooks at 1731809040067 (+7 ms)Closed at 1731809040067 2024-11-17T02:04:00,068 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1731809034900.4748cc4c5ad3838a3d0568cee797c169. 2024-11-17T02:04:00,068 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-17T02:04:00,068 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-17T02:04:00,068 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6fd33da09f9d938215ac0e7127ad70d5, disabling compactions & flushes 2024-11-17T02:04:00,068 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1731809034900.6fd33da09f9d938215ac0e7127ad70d5. 2024-11-17T02:04:00,068 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731809039987Running coprocessor pre-close hooks at 1731809039987Disabling compacts and flushes for region at 1731809039987Disabling writes for close at 1731809039987Obtaining lock to block concurrent updates at 1731809039987Preparing flush snapshotting stores in 1588230740 at 1731809039987Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2708, getHeapSize=5488, getOffHeapSize=0, getCellsCount=24 at 1731809039994 (+7 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731809039996 (+2 ms)Flushing 1588230740/info: creating writer at 1731809039996Flushing 1588230740/info: appending metadata at 1731809040008 (+12 ms)Flushing 1588230740/info: closing flushed file at 1731809040008Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@792fdd62: reopening flushed file at 1731809040045 (+37 ms)Finished flush of dataSize ~2.64 KB/2708, heapSize ~4.66 KB/4768, currentSize=0 B/0 for 1588230740 in 70ms, sequenceid=71, compaction requested=false at 1731809040057 (+12 ms)Writing region close event to WAL at 1731809040063 (+6 ms)Running coprocessor post-close hooks at 1731809040068 (+5 ms)Closed at 1731809040068 2024-11-17T02:04:00,068 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1731809034900.6fd33da09f9d938215ac0e7127ad70d5. 2024-11-17T02:04:00,068 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1731809034900.6fd33da09f9d938215ac0e7127ad70d5. after waiting 0 ms 2024-11-17T02:04:00,068 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-17T02:04:00,068 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1731809034900.6fd33da09f9d938215ac0e7127ad70d5. 2024-11-17T02:04:00,074 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/6fd33da09f9d938215ac0e7127ad70d5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T02:04:00,074 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/81f322eb57dcb42c3b4269a738bafd6d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T02:04:00,075 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1731809034900.6fd33da09f9d938215ac0e7127ad70d5. 2024-11-17T02:04:00,075 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1731809034900.81f322eb57dcb42c3b4269a738bafd6d. 2024-11-17T02:04:00,075 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6fd33da09f9d938215ac0e7127ad70d5: Waiting for close lock at 1731809040068Running coprocessor pre-close hooks at 1731809040068Disabling compacts and flushes for region at 1731809040068Disabling writes for close at 1731809040068Writing region close event to WAL at 1731809040071 (+3 ms)Running coprocessor post-close hooks at 1731809040074 (+3 ms)Closed at 1731809040075 (+1 ms) 2024-11-17T02:04:00,075 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 81f322eb57dcb42c3b4269a738bafd6d: Waiting for close lock at 1731809040064Running coprocessor pre-close hooks at 1731809040064Disabling compacts and flushes for region at 1731809040064Disabling writes for close at 1731809040065 (+1 ms)Writing region close event to WAL at 1731809040070 (+5 ms)Running coprocessor post-close hooks at 1731809040075 (+5 ms)Closed at 1731809040075 2024-11-17T02:04:00,075 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1731809034900.6fd33da09f9d938215ac0e7127ad70d5. 2024-11-17T02:04:00,075 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1731809034900.81f322eb57dcb42c3b4269a738bafd6d. 2024-11-17T02:04:00,075 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 15c64fab21a3ca03ec0a7afabc19cdc9, disabling compactions & flushes 2024-11-17T02:04:00,075 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9d86e1fe125f78846dcaa5b9921c72db, disabling compactions & flushes 2024-11-17T02:04:00,075 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1731809034900.15c64fab21a3ca03ec0a7afabc19cdc9. 2024-11-17T02:04:00,075 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1731809034900.9d86e1fe125f78846dcaa5b9921c72db. 2024-11-17T02:04:00,075 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1731809034900.15c64fab21a3ca03ec0a7afabc19cdc9. 2024-11-17T02:04:00,075 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1731809034900.9d86e1fe125f78846dcaa5b9921c72db. 2024-11-17T02:04:00,075 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1731809034900.15c64fab21a3ca03ec0a7afabc19cdc9. after waiting 0 ms 2024-11-17T02:04:00,075 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1731809034900.9d86e1fe125f78846dcaa5b9921c72db. after waiting 0 ms 2024-11-17T02:04:00,075 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1731809034900.15c64fab21a3ca03ec0a7afabc19cdc9. 2024-11-17T02:04:00,075 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1731809034900.9d86e1fe125f78846dcaa5b9921c72db. 2024-11-17T02:04:00,079 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/15c64fab21a3ca03ec0a7afabc19cdc9/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T02:04:00,079 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/9d86e1fe125f78846dcaa5b9921c72db/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T02:04:00,080 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1731809034900.15c64fab21a3ca03ec0a7afabc19cdc9. 2024-11-17T02:04:00,080 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 15c64fab21a3ca03ec0a7afabc19cdc9: Waiting for close lock at 1731809040075Running coprocessor pre-close hooks at 1731809040075Disabling compacts and flushes for region at 1731809040075Disabling writes for close at 1731809040075Writing region close event to WAL at 1731809040075Running coprocessor post-close hooks at 1731809040079 (+4 ms)Closed at 1731809040079 2024-11-17T02:04:00,080 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1731809034900.9d86e1fe125f78846dcaa5b9921c72db. 2024-11-17T02:04:00,080 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9d86e1fe125f78846dcaa5b9921c72db: Waiting for close lock at 1731809040075Running coprocessor pre-close hooks at 1731809040075Disabling compacts and flushes for region at 1731809040075Disabling writes for close at 1731809040075Writing region close event to WAL at 1731809040075Running coprocessor post-close hooks at 1731809040080 (+5 ms)Closed at 1731809040080 2024-11-17T02:04:00,080 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1731809034900.15c64fab21a3ca03ec0a7afabc19cdc9. 2024-11-17T02:04:00,080 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1731809034900.9d86e1fe125f78846dcaa5b9921c72db. 2024-11-17T02:04:00,080 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 77bd97fde6bd169f3e20553b45a65e2c, disabling compactions & flushes 2024-11-17T02:04:00,080 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1731809034900.77bd97fde6bd169f3e20553b45a65e2c. 2024-11-17T02:04:00,080 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 112f5bbe15f72793585db9ba98950976, disabling compactions & flushes 2024-11-17T02:04:00,080 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1731809034900.112f5bbe15f72793585db9ba98950976. 2024-11-17T02:04:00,080 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1731809034900.77bd97fde6bd169f3e20553b45a65e2c. 2024-11-17T02:04:00,080 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1731809034900.112f5bbe15f72793585db9ba98950976. 2024-11-17T02:04:00,080 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1731809034900.77bd97fde6bd169f3e20553b45a65e2c. after waiting 0 ms 2024-11-17T02:04:00,080 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1731809034900.77bd97fde6bd169f3e20553b45a65e2c. 2024-11-17T02:04:00,080 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1731809034900.112f5bbe15f72793585db9ba98950976. after waiting 0 ms 2024-11-17T02:04:00,080 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1731809034900.112f5bbe15f72793585db9ba98950976. 2024-11-17T02:04:00,084 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/112f5bbe15f72793585db9ba98950976/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T02:04:00,085 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1731809034900.112f5bbe15f72793585db9ba98950976. 2024-11-17T02:04:00,085 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 112f5bbe15f72793585db9ba98950976: Waiting for close lock at 1731809040080Running coprocessor pre-close hooks at 1731809040080Disabling compacts and flushes for region at 1731809040080Disabling writes for close at 1731809040080Writing region close event to WAL at 1731809040080Running coprocessor post-close hooks at 1731809040085 (+5 ms)Closed at 1731809040085 2024-11-17T02:04:00,085 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1731809034900.112f5bbe15f72793585db9ba98950976. 2024-11-17T02:04:00,085 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1056118c93716b24d18f749826cf316a, disabling compactions & flushes 2024-11-17T02:04:00,085 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1731809034900.1056118c93716b24d18f749826cf316a. 2024-11-17T02:04:00,085 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1731809034900.1056118c93716b24d18f749826cf316a. 2024-11-17T02:04:00,085 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1731809034900.1056118c93716b24d18f749826cf316a. after waiting 0 ms 2024-11-17T02:04:00,085 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1731809034900.1056118c93716b24d18f749826cf316a. 2024-11-17T02:04:00,086 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/77bd97fde6bd169f3e20553b45a65e2c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T02:04:00,086 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1731809034900.77bd97fde6bd169f3e20553b45a65e2c. 2024-11-17T02:04:00,086 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 77bd97fde6bd169f3e20553b45a65e2c: Waiting for close lock at 1731809040080Running coprocessor pre-close hooks at 1731809040080Disabling compacts and flushes for region at 1731809040080Disabling writes for close at 1731809040080Writing region close event to WAL at 1731809040080Running coprocessor post-close hooks at 1731809040086 (+6 ms)Closed at 1731809040086 2024-11-17T02:04:00,087 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1731809034900.77bd97fde6bd169f3e20553b45a65e2c. 2024-11-17T02:04:00,087 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 05534a4a027a1fdc90227b7a2e681933, disabling compactions & flushes 2024-11-17T02:04:00,087 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1731809034900.05534a4a027a1fdc90227b7a2e681933. 2024-11-17T02:04:00,087 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1731809034900.05534a4a027a1fdc90227b7a2e681933. 2024-11-17T02:04:00,087 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1731809034900.05534a4a027a1fdc90227b7a2e681933. after waiting 0 ms 2024-11-17T02:04:00,087 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1731809034900.05534a4a027a1fdc90227b7a2e681933. 2024-11-17T02:04:00,090 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/1056118c93716b24d18f749826cf316a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T02:04:00,091 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1731809034900.1056118c93716b24d18f749826cf316a. 2024-11-17T02:04:00,091 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/05534a4a027a1fdc90227b7a2e681933/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T02:04:00,091 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1056118c93716b24d18f749826cf316a: Waiting for close lock at 1731809040085Running coprocessor pre-close hooks at 1731809040085Disabling compacts and flushes for region at 1731809040085Disabling writes for close at 1731809040085Writing region close event to WAL at 1731809040086 (+1 ms)Running coprocessor post-close hooks at 1731809040091 (+5 ms)Closed at 1731809040091 2024-11-17T02:04:00,091 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1731809034900.1056118c93716b24d18f749826cf316a. 2024-11-17T02:04:00,091 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a17ac9e270344fa2752f51f8196693b2, disabling compactions & flushes 2024-11-17T02:04:00,092 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1731809034900.a17ac9e270344fa2752f51f8196693b2. 2024-11-17T02:04:00,092 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1731809034900.a17ac9e270344fa2752f51f8196693b2. 2024-11-17T02:04:00,092 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1731809034900.a17ac9e270344fa2752f51f8196693b2. after waiting 0 ms 2024-11-17T02:04:00,092 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1731809034900.05534a4a027a1fdc90227b7a2e681933. 2024-11-17T02:04:00,092 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1731809034900.a17ac9e270344fa2752f51f8196693b2. 2024-11-17T02:04:00,092 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 05534a4a027a1fdc90227b7a2e681933: Waiting for close lock at 1731809040087Running coprocessor pre-close hooks at 1731809040087Disabling compacts and flushes for region at 1731809040087Disabling writes for close at 1731809040087Writing region close event to WAL at 1731809040087Running coprocessor post-close hooks at 1731809040092 (+5 ms)Closed at 1731809040092 2024-11-17T02:04:00,092 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1731809034900.05534a4a027a1fdc90227b7a2e681933. 2024-11-17T02:04:00,092 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing de9924be32211cddceef43e5b4866dc8, disabling compactions & flushes 2024-11-17T02:04:00,092 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1731809034900.de9924be32211cddceef43e5b4866dc8. 2024-11-17T02:04:00,092 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1731809034900.de9924be32211cddceef43e5b4866dc8. 2024-11-17T02:04:00,092 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1731809034900.de9924be32211cddceef43e5b4866dc8. after waiting 0 ms 2024-11-17T02:04:00,092 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1731809034900.de9924be32211cddceef43e5b4866dc8. 2024-11-17T02:04:00,096 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/a17ac9e270344fa2752f51f8196693b2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T02:04:00,096 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/de9924be32211cddceef43e5b4866dc8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T02:04:00,098 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1731809034900.a17ac9e270344fa2752f51f8196693b2. 2024-11-17T02:04:00,098 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1731809034900.de9924be32211cddceef43e5b4866dc8. 2024-11-17T02:04:00,098 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a17ac9e270344fa2752f51f8196693b2: Waiting for close lock at 1731809040091Running coprocessor pre-close hooks at 1731809040091Disabling compacts and flushes for region at 1731809040091Disabling writes for close at 1731809040092 (+1 ms)Writing region close event to WAL at 1731809040092Running coprocessor post-close hooks at 1731809040098 (+6 ms)Closed at 1731809040098 2024-11-17T02:04:00,098 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for de9924be32211cddceef43e5b4866dc8: Waiting for close lock at 1731809040092Running coprocessor pre-close hooks at 1731809040092Disabling compacts and flushes for region at 1731809040092Disabling writes for close at 1731809040092Writing region close event to WAL at 1731809040092Running coprocessor post-close hooks at 1731809040098 (+6 ms)Closed at 1731809040098 2024-11-17T02:04:00,098 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1731809034900.de9924be32211cddceef43e5b4866dc8. 2024-11-17T02:04:00,098 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1731809034900.a17ac9e270344fa2752f51f8196693b2. 2024-11-17T02:04:00,098 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2f4fe90ec03839f5d26acd9d02569e56, disabling compactions & flushes 2024-11-17T02:04:00,098 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 74108f0b014075d4993f0eee396e39e5, disabling compactions & flushes 2024-11-17T02:04:00,098 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1731809034900.2f4fe90ec03839f5d26acd9d02569e56. 2024-11-17T02:04:00,098 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1731809034900.74108f0b014075d4993f0eee396e39e5. 2024-11-17T02:04:00,098 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1731809034900.2f4fe90ec03839f5d26acd9d02569e56. 2024-11-17T02:04:00,098 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1731809034900.74108f0b014075d4993f0eee396e39e5. 2024-11-17T02:04:00,098 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1731809034900.2f4fe90ec03839f5d26acd9d02569e56. after waiting 0 ms 2024-11-17T02:04:00,098 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1731809034900.2f4fe90ec03839f5d26acd9d02569e56. 2024-11-17T02:04:00,099 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1731809034900.74108f0b014075d4993f0eee396e39e5. after waiting 0 ms 2024-11-17T02:04:00,099 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1731809034900.74108f0b014075d4993f0eee396e39e5. 2024-11-17T02:04:00,103 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/74108f0b014075d4993f0eee396e39e5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T02:04:00,103 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/2f4fe90ec03839f5d26acd9d02569e56/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T02:04:00,104 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1731809034900.74108f0b014075d4993f0eee396e39e5. 2024-11-17T02:04:00,104 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1731809034900.2f4fe90ec03839f5d26acd9d02569e56. 2024-11-17T02:04:00,104 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 74108f0b014075d4993f0eee396e39e5: Waiting for close lock at 1731809040098Running coprocessor pre-close hooks at 1731809040098Disabling compacts and flushes for region at 1731809040098Disabling writes for close at 1731809040099 (+1 ms)Writing region close event to WAL at 1731809040099Running coprocessor post-close hooks at 1731809040104 (+5 ms)Closed at 1731809040104 2024-11-17T02:04:00,104 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2f4fe90ec03839f5d26acd9d02569e56: Waiting for close lock at 1731809040098Running coprocessor pre-close hooks at 1731809040098Disabling compacts and flushes for region at 1731809040098Disabling writes for close at 1731809040098Writing region close event to WAL at 1731809040099 (+1 ms)Running coprocessor post-close hooks at 1731809040104 (+5 ms)Closed at 1731809040104 2024-11-17T02:04:00,104 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1731809034900.74108f0b014075d4993f0eee396e39e5. 2024-11-17T02:04:00,104 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1731809034900.2f4fe90ec03839f5d26acd9d02569e56. 2024-11-17T02:04:00,104 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b7934444caa228b9e5d6d7b28b59a796, disabling compactions & flushes 2024-11-17T02:04:00,104 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 037b7a93730957425bc54697b3594bb2, disabling compactions & flushes 2024-11-17T02:04:00,104 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1731809034900.b7934444caa228b9e5d6d7b28b59a796. 2024-11-17T02:04:00,104 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1731809034900.037b7a93730957425bc54697b3594bb2. 2024-11-17T02:04:00,104 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1731809034900.b7934444caa228b9e5d6d7b28b59a796. 2024-11-17T02:04:00,104 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1731809034900.037b7a93730957425bc54697b3594bb2. 2024-11-17T02:04:00,104 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1731809034900.037b7a93730957425bc54697b3594bb2. after waiting 0 ms 2024-11-17T02:04:00,104 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1731809034900.b7934444caa228b9e5d6d7b28b59a796. after waiting 0 ms 2024-11-17T02:04:00,104 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1731809034900.037b7a93730957425bc54697b3594bb2. 2024-11-17T02:04:00,104 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1731809034900.b7934444caa228b9e5d6d7b28b59a796. 2024-11-17T02:04:00,109 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/037b7a93730957425bc54697b3594bb2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T02:04:00,109 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/data/default/test/b7934444caa228b9e5d6d7b28b59a796/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T02:04:00,109 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1731809034900.037b7a93730957425bc54697b3594bb2. 2024-11-17T02:04:00,109 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 037b7a93730957425bc54697b3594bb2: Waiting for close lock at 1731809040104Running coprocessor pre-close hooks at 1731809040104Disabling compacts and flushes for region at 1731809040104Disabling writes for close at 1731809040104Writing region close event to WAL at 1731809040105 (+1 ms)Running coprocessor post-close hooks at 1731809040109 (+4 ms)Closed at 1731809040109 2024-11-17T02:04:00,109 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1731809034900.b7934444caa228b9e5d6d7b28b59a796. 2024-11-17T02:04:00,110 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b7934444caa228b9e5d6d7b28b59a796: Waiting for close lock at 1731809040104Running coprocessor pre-close hooks at 1731809040104Disabling compacts and flushes for region at 1731809040104Disabling writes for close at 1731809040104Writing region close event to WAL at 1731809040105 (+1 ms)Running coprocessor post-close hooks at 1731809040109 (+4 ms)Closed at 1731809040109 2024-11-17T02:04:00,110 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1731809034900.037b7a93730957425bc54697b3594bb2. 2024-11-17T02:04:00,110 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1731809034900.b7934444caa228b9e5d6d7b28b59a796. 2024-11-17T02:04:00,183 INFO [RS:0;c6c0d7a0a7c0:33127 {}] regionserver.HRegionServer(976): stopping server c6c0d7a0a7c0,33127,1731809032263; all regions closed. 2024-11-17T02:04:00,187 INFO [RS:1;c6c0d7a0a7c0:45037 {}] regionserver.HRegionServer(976): stopping server c6c0d7a0a7c0,45037,1731809032394; all regions closed. 2024-11-17T02:04:00,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741833_1009 (size=7064) 2024-11-17T02:04:00,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741833_1009 (size=7064) 2024-11-17T02:04:00,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741833_1009 (size=7064) 2024-11-17T02:04:00,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741865_1041 (size=4192) 2024-11-17T02:04:00,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741865_1041 (size=4192) 2024-11-17T02:04:00,190 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/oldWALs 2024-11-17T02:04:00,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741865_1041 (size=4192) 2024-11-17T02:04:00,191 INFO [RS:0;c6c0d7a0a7c0:33127 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c6c0d7a0a7c0%2C33127%2C1731809032263:(num 1731809034116) 2024-11-17T02:04:00,191 DEBUG [RS:0;c6c0d7a0a7c0:33127 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T02:04:00,191 INFO [RS:0;c6c0d7a0a7c0:33127 {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T02:04:00,191 INFO [RS:0;c6c0d7a0a7c0:33127 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-17T02:04:00,191 INFO [RS:0;c6c0d7a0a7c0:33127 {}] hbase.ChoreService(370): Chore service for: regionserver/c6c0d7a0a7c0:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-17T02:04:00,191 INFO [RS:0;c6c0d7a0a7c0:33127 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-17T02:04:00,191 INFO [RS:0;c6c0d7a0a7c0:33127 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-17T02:04:00,191 INFO [regionserver/c6c0d7a0a7c0:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-17T02:04:00,192 INFO [RS:0;c6c0d7a0a7c0:33127 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-17T02:04:00,192 INFO [RS:0;c6c0d7a0a7c0:33127 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-17T02:04:00,192 INFO [RS:0;c6c0d7a0a7c0:33127 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33127 2024-11-17T02:04:00,193 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/oldWALs 2024-11-17T02:04:00,193 INFO [RS:1;c6c0d7a0a7c0:45037 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c6c0d7a0a7c0%2C45037%2C1731809032394.meta:.meta(num 1731809038035) 2024-11-17T02:04:00,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741834_1010 (size=7074) 2024-11-17T02:04:00,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741834_1010 (size=7074) 2024-11-17T02:04:00,197 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/WALs/c6c0d7a0a7c0,45037,1731809032394/c6c0d7a0a7c0%2C45037%2C1731809032394.1731809034116 not finished, retry = 0 2024-11-17T02:04:00,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741834_1010 (size=7074) 2024-11-17T02:04:00,219 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-17T02:04:00,219 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c6c0d7a0a7c0,33127,1731809032263 2024-11-17T02:04:00,219 INFO [RS:0;c6c0d7a0a7c0:33127 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-17T02:04:00,227 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c6c0d7a0a7c0,33127,1731809032263] 2024-11-17T02:04:00,235 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c6c0d7a0a7c0,33127,1731809032263 already deleted, retry=false 2024-11-17T02:04:00,236 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c6c0d7a0a7c0,33127,1731809032263 expired; onlineServers=1 2024-11-17T02:04:00,304 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/oldWALs 2024-11-17T02:04:00,304 INFO [RS:1;c6c0d7a0a7c0:45037 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c6c0d7a0a7c0%2C45037%2C1731809032394:(num 1731809034116) 2024-11-17T02:04:00,304 DEBUG [RS:1;c6c0d7a0a7c0:45037 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T02:04:00,304 INFO [RS:1;c6c0d7a0a7c0:45037 {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T02:04:00,304 INFO [RS:1;c6c0d7a0a7c0:45037 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-17T02:04:00,305 INFO [RS:1;c6c0d7a0a7c0:45037 {}] hbase.ChoreService(370): Chore service for: regionserver/c6c0d7a0a7c0:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-17T02:04:00,305 INFO [RS:1;c6c0d7a0a7c0:45037 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-17T02:04:00,305 INFO [regionserver/c6c0d7a0a7c0:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-17T02:04:00,305 INFO [RS:1;c6c0d7a0a7c0:45037 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45037 2024-11-17T02:04:00,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-17T02:04:00,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c6c0d7a0a7c0,45037,1731809032394 2024-11-17T02:04:00,319 INFO [RS:1;c6c0d7a0a7c0:45037 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-17T02:04:00,319 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c6c0d7a0a7c0,45037,1731809032394] 2024-11-17T02:04:00,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T02:04:00,327 INFO [RS:0;c6c0d7a0a7c0:33127 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-17T02:04:00,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33127-0x10147d245010001, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T02:04:00,327 INFO [RS:0;c6c0d7a0a7c0:33127 {}] regionserver.HRegionServer(1031): Exiting; stopping=c6c0d7a0a7c0,33127,1731809032263; zookeeper connection closed. 2024-11-17T02:04:00,327 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@367e841 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@367e841 2024-11-17T02:04:00,335 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c6c0d7a0a7c0,45037,1731809032394 already deleted, retry=false 2024-11-17T02:04:00,335 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c6c0d7a0a7c0,45037,1731809032394 expired; onlineServers=0 2024-11-17T02:04:00,336 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c6c0d7a0a7c0,32803,1731809031530' ***** 2024-11-17T02:04:00,336 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-17T02:04:00,336 INFO [M:0;c6c0d7a0a7c0:32803 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-17T02:04:00,336 INFO [M:0;c6c0d7a0a7c0:32803 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-17T02:04:00,336 DEBUG [M:0;c6c0d7a0a7c0:32803 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-17T02:04:00,336 DEBUG [M:0;c6c0d7a0a7c0:32803 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-17T02:04:00,336 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster-HFileCleaner.large.0-1731809033647 {}] cleaner.HFileCleaner(306): Exit Thread[master/c6c0d7a0a7c0:0:becomeActiveMaster-HFileCleaner.large.0-1731809033647,5,FailOnTimeoutGroup] 2024-11-17T02:04:00,336 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster-HFileCleaner.small.0-1731809033648 {}] cleaner.HFileCleaner(306): Exit Thread[master/c6c0d7a0a7c0:0:becomeActiveMaster-HFileCleaner.small.0-1731809033648,5,FailOnTimeoutGroup] 2024-11-17T02:04:00,336 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-17T02:04:00,337 INFO [M:0;c6c0d7a0a7c0:32803 {}] hbase.ChoreService(370): Chore service for: master/c6c0d7a0a7c0:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-17T02:04:00,337 INFO [M:0;c6c0d7a0a7c0:32803 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-17T02:04:00,337 DEBUG [M:0;c6c0d7a0a7c0:32803 {}] master.HMaster(1795): Stopping service threads 2024-11-17T02:04:00,337 INFO [M:0;c6c0d7a0a7c0:32803 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-17T02:04:00,337 INFO [M:0;c6c0d7a0a7c0:32803 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T02:04:00,337 INFO [M:0;c6c0d7a0a7c0:32803 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-17T02:04:00,338 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-17T02:04:00,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-17T02:04:00,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T02:04:00,344 DEBUG [M:0;c6c0d7a0a7c0:32803 {}] zookeeper.ZKUtil(347): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-17T02:04:00,344 WARN [M:0;c6c0d7a0a7c0:32803 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-17T02:04:00,345 INFO [M:0;c6c0d7a0a7c0:32803 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/.lastflushedseqids 2024-11-17T02:04:00,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741870_1046 (size=1) 2024-11-17T02:04:00,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741870_1046 (size=1) 2024-11-17T02:04:00,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741870_1046 (size=1) 2024-11-17T02:04:00,360 INFO [M:0;c6c0d7a0a7c0:32803 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-17T02:04:00,361 INFO [M:0;c6c0d7a0a7c0:32803 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-17T02:04:00,361 DEBUG [M:0;c6c0d7a0a7c0:32803 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-17T02:04:00,361 INFO [M:0;c6c0d7a0a7c0:32803 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T02:04:00,361 DEBUG [M:0;c6c0d7a0a7c0:32803 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T02:04:00,361 DEBUG [M:0;c6c0d7a0a7c0:32803 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-17T02:04:00,361 DEBUG [M:0;c6c0d7a0a7c0:32803 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T02:04:00,361 INFO [M:0;c6c0d7a0a7c0:32803 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=250.07 KB heapSize=303.91 KB 2024-11-17T02:04:00,381 DEBUG [M:0;c6c0d7a0a7c0:32803 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6beb5f498b0c4e908eb47f1aac916be6 is 82, key is hbase:meta,,1/info:regioninfo/1731809038348/Put/seqid=0 2024-11-17T02:04:00,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741871_1047 (size=6071) 2024-11-17T02:04:00,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741871_1047 (size=6071) 2024-11-17T02:04:00,388 INFO [M:0;c6c0d7a0a7c0:32803 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6beb5f498b0c4e908eb47f1aac916be6 2024-11-17T02:04:00,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741871_1047 (size=6071) 2024-11-17T02:04:00,414 DEBUG [M:0;c6c0d7a0a7c0:32803 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/389bac00eda64a3687f92350bf68d792 is 2987, key is \x00\x00\x00\x00\x00\x00\x009/proc:d/1731809039945/Put/seqid=0 2024-11-17T02:04:00,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741872_1048 (size=12799) 2024-11-17T02:04:00,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741872_1048 (size=12799) 2024-11-17T02:04:00,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741872_1048 (size=12799) 2024-11-17T02:04:00,426 INFO [M:0;c6c0d7a0a7c0:32803 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=248.80 KB at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/389bac00eda64a3687f92350bf68d792 2024-11-17T02:04:00,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T02:04:00,427 INFO [RS:1;c6c0d7a0a7c0:45037 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-17T02:04:00,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45037-0x10147d245010002, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T02:04:00,427 INFO [RS:1;c6c0d7a0a7c0:45037 {}] regionserver.HRegionServer(1031): Exiting; stopping=c6c0d7a0a7c0,45037,1731809032394; zookeeper connection closed. 2024-11-17T02:04:00,427 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4ec3b0ec {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4ec3b0ec 2024-11-17T02:04:00,428 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-17T02:04:00,456 DEBUG [M:0;c6c0d7a0a7c0:32803 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5275b173df6d40cf9fb06f02ced57053 is 69, key is c6c0d7a0a7c0,33127,1731809032263/rs:state/1731809033787/Put/seqid=0 2024-11-17T02:04:00,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741873_1049 (size=5447) 2024-11-17T02:04:00,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741873_1049 (size=5447) 2024-11-17T02:04:00,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741873_1049 (size=5447) 2024-11-17T02:04:00,463 INFO [M:0;c6c0d7a0a7c0:32803 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5275b173df6d40cf9fb06f02ced57053 2024-11-17T02:04:00,469 INFO [M:0;c6c0d7a0a7c0:32803 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5275b173df6d40cf9fb06f02ced57053 2024-11-17T02:04:00,483 DEBUG [M:0;c6c0d7a0a7c0:32803 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ad52fcc190d0476cbaa46436437af7f9 is 52, key is load_balancer_on/state:d/1731809037241/Put/seqid=0 2024-11-17T02:04:00,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741874_1050 (size=5058) 2024-11-17T02:04:00,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741874_1050 (size=5058) 2024-11-17T02:04:00,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741874_1050 (size=5058) 2024-11-17T02:04:00,491 INFO [M:0;c6c0d7a0a7c0:32803 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ad52fcc190d0476cbaa46436437af7f9 2024-11-17T02:04:00,498 DEBUG [M:0;c6c0d7a0a7c0:32803 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6beb5f498b0c4e908eb47f1aac916be6 as hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6beb5f498b0c4e908eb47f1aac916be6 2024-11-17T02:04:00,506 INFO [M:0;c6c0d7a0a7c0:32803 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6beb5f498b0c4e908eb47f1aac916be6, entries=14, sequenceid=575, filesize=5.9 K 2024-11-17T02:04:00,507 DEBUG [M:0;c6c0d7a0a7c0:32803 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/389bac00eda64a3687f92350bf68d792 as hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/389bac00eda64a3687f92350bf68d792 2024-11-17T02:04:00,513 INFO [M:0;c6c0d7a0a7c0:32803 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/389bac00eda64a3687f92350bf68d792, entries=79, sequenceid=575, filesize=12.5 K 2024-11-17T02:04:00,514 DEBUG [M:0;c6c0d7a0a7c0:32803 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5275b173df6d40cf9fb06f02ced57053 as hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5275b173df6d40cf9fb06f02ced57053 2024-11-17T02:04:00,521 INFO [M:0;c6c0d7a0a7c0:32803 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5275b173df6d40cf9fb06f02ced57053 2024-11-17T02:04:00,521 INFO [M:0;c6c0d7a0a7c0:32803 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5275b173df6d40cf9fb06f02ced57053, entries=3, sequenceid=575, filesize=5.3 K 2024-11-17T02:04:00,522 DEBUG [M:0;c6c0d7a0a7c0:32803 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ad52fcc190d0476cbaa46436437af7f9 as hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/ad52fcc190d0476cbaa46436437af7f9 2024-11-17T02:04:00,528 INFO [M:0;c6c0d7a0a7c0:32803 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40353/user/jenkins/test-data/9755fb59-8c30-4f7b-979f-53afdee520fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/ad52fcc190d0476cbaa46436437af7f9, entries=1, sequenceid=575, filesize=4.9 K 2024-11-17T02:04:00,530 INFO [M:0;c6c0d7a0a7c0:32803 {}] regionserver.HRegion(3140): Finished flush of dataSize ~250.07 KB/256072, heapSize ~303.85 KB/311144, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 168ms, sequenceid=575, compaction requested=false 2024-11-17T02:04:00,534 INFO [M:0;c6c0d7a0a7c0:32803 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T02:04:00,534 DEBUG [M:0;c6c0d7a0a7c0:32803 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731809040361Disabling compacts and flushes for region at 1731809040361Disabling writes for close at 1731809040361Obtaining lock to block concurrent updates at 1731809040361Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731809040361Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=256072, getHeapSize=311144, getOffHeapSize=0, getCellsCount=714 at 1731809040362 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731809040362Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731809040362Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731809040380 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731809040380Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731809040393 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731809040413 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731809040413Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731809040434 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731809040455 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731809040455Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731809040469 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731809040483 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731809040483Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75650fdd: reopening flushed file at 1731809040497 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1be89586: reopening flushed file at 1731809040506 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19373874: reopening flushed file at 1731809040513 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@302a4a76: reopening flushed file at 1731809040521 (+8 ms)Finished flush of dataSize ~250.07 KB/256072, heapSize ~303.85 KB/311144, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 168ms, sequenceid=575, compaction requested=false at 1731809040530 (+9 ms)Writing region close event to WAL at 1731809040534 (+4 ms)Closed at 1731809040534 2024-11-17T02:04:00,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33655 is added to blk_1073741830_1006 (size=294456) 2024-11-17T02:04:00,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36067 is added to blk_1073741830_1006 (size=294456) 2024-11-17T02:04:00,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36455 is added to blk_1073741830_1006 (size=294456) 2024-11-17T02:04:00,538 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-17T02:04:00,538 INFO [M:0;c6c0d7a0a7c0:32803 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-17T02:04:00,538 INFO [M:0;c6c0d7a0a7c0:32803 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:32803 2024-11-17T02:04:00,538 INFO [M:0;c6c0d7a0a7c0:32803 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-17T02:04:00,661 INFO [M:0;c6c0d7a0a7c0:32803 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-17T02:04:00,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T02:04:00,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32803-0x10147d245010000, quorum=127.0.0.1:53449, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T02:04:00,694 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3c1f701{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-17T02:04:00,699 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3d599bb4{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-17T02:04:00,699 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-17T02:04:00,699 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@417efacb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-17T02:04:00,700 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@290e2eec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/hadoop.log.dir/,STOPPED} 2024-11-17T02:04:00,705 WARN [BP-927759803-172.17.0.2-1731809027758 heartbeating to localhost/127.0.0.1:40353 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-17T02:04:00,705 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-17T02:04:00,705 WARN [BP-927759803-172.17.0.2-1731809027758 heartbeating to localhost/127.0.0.1:40353 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-927759803-172.17.0.2-1731809027758 (Datanode Uuid 4107a711-3cdd-4db3-a24e-f4f492515acf) service to localhost/127.0.0.1:40353 2024-11-17T02:04:00,705 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-17T02:04:00,706 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/cluster_e712193e-aef2-a358-3728-f7558b6dc143/data/data5/current/BP-927759803-172.17.0.2-1731809027758 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-17T02:04:00,707 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/cluster_e712193e-aef2-a358-3728-f7558b6dc143/data/data6/current/BP-927759803-172.17.0.2-1731809027758 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-17T02:04:00,707 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-17T02:04:00,709 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@34fcb585{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-17T02:04:00,709 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@389bb6f7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-17T02:04:00,709 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-17T02:04:00,709 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1b0a32ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-17T02:04:00,709 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@29652cf4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/hadoop.log.dir/,STOPPED} 2024-11-17T02:04:00,710 WARN [BP-927759803-172.17.0.2-1731809027758 heartbeating to localhost/127.0.0.1:40353 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-17T02:04:00,710 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-17T02:04:00,710 WARN [BP-927759803-172.17.0.2-1731809027758 heartbeating to localhost/127.0.0.1:40353 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-927759803-172.17.0.2-1731809027758 (Datanode Uuid 92058095-02df-4731-a81d-1580a9bbc0f1) service to localhost/127.0.0.1:40353 2024-11-17T02:04:00,710 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-17T02:04:00,711 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/cluster_e712193e-aef2-a358-3728-f7558b6dc143/data/data3/current/BP-927759803-172.17.0.2-1731809027758 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-17T02:04:00,711 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/cluster_e712193e-aef2-a358-3728-f7558b6dc143/data/data4/current/BP-927759803-172.17.0.2-1731809027758 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-17T02:04:00,711 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-17T02:04:00,713 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@79b30fb0{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-17T02:04:00,714 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@548cd725{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-17T02:04:00,714 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-17T02:04:00,714 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@474d518f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-17T02:04:00,714 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3a300759{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/hadoop.log.dir/,STOPPED} 2024-11-17T02:04:00,716 WARN [BP-927759803-172.17.0.2-1731809027758 heartbeating to localhost/127.0.0.1:40353 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-17T02:04:00,716 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-17T02:04:00,716 WARN [BP-927759803-172.17.0.2-1731809027758 heartbeating to localhost/127.0.0.1:40353 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-927759803-172.17.0.2-1731809027758 (Datanode Uuid b4ea61d2-621c-4f78-9946-a6c292a6b2a5) service to localhost/127.0.0.1:40353 2024-11-17T02:04:00,716 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-17T02:04:00,717 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/cluster_e712193e-aef2-a358-3728-f7558b6dc143/data/data1/current/BP-927759803-172.17.0.2-1731809027758 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-17T02:04:00,717 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/cluster_e712193e-aef2-a358-3728-f7558b6dc143/data/data2/current/BP-927759803-172.17.0.2-1731809027758 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-17T02:04:00,717 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-17T02:04:00,726 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@64418798{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-17T02:04:00,727 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-17T02:04:00,727 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-17T02:04:00,727 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-17T02:04:00,727 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/111593c5-6e73-6125-edea-0b0898321846/hadoop.log.dir/,STOPPED} 2024-11-17T02:04:00,737 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-17T02:04:00,785 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down