2024-12-11 22:40:51,550 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-12-11 22:40:51,568 main DEBUG Took 0.010663 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-11 22:40:51,568 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-11 22:40:51,568 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-11 22:40:51,569 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-11 22:40:51,570 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 22:40:51,577 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-11 22:40:51,588 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 22:40:51,589 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 22:40:51,590 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 22:40:51,591 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 22:40:51,591 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 22:40:51,592 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 22:40:51,593 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 22:40:51,593 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 22:40:51,594 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 22:40:51,594 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 22:40:51,595 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 22:40:51,596 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 22:40:51,596 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 22:40:51,597 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 22:40:51,597 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 22:40:51,598 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 22:40:51,598 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 22:40:51,599 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 22:40:51,599 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 22:40:51,600 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 22:40:51,600 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 22:40:51,601 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 22:40:51,601 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 22:40:51,602 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 22:40:51,602 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 22:40:51,602 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-11 22:40:51,604 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 22:40:51,606 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-11 22:40:51,608 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-11 22:40:51,609 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-11 22:40:51,610 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-11 22:40:51,611 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-11 22:40:51,620 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-11 22:40:51,624 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-11 22:40:51,626 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-11 22:40:51,626 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-11 22:40:51,627 main DEBUG createAppenders(={Console}) 2024-12-11 22:40:51,628 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-12-11 22:40:51,628 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-12-11 22:40:51,628 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-12-11 22:40:51,629 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-11 22:40:51,629 main DEBUG OutputStream closed 2024-12-11 22:40:51,630 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-11 22:40:51,630 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-11 22:40:51,630 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-12-11 22:40:51,710 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-11 22:40:51,713 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-11 22:40:51,714 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-11 22:40:51,714 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-11 22:40:51,715 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-11 22:40:51,715 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-11 22:40:51,716 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-11 22:40:51,716 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-11 22:40:51,716 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-11 22:40:51,717 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-11 22:40:51,717 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-11 22:40:51,717 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-11 22:40:51,717 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-11 22:40:51,718 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-11 22:40:51,718 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-11 22:40:51,718 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-11 22:40:51,718 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-11 22:40:51,719 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-11 22:40:51,721 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-11 22:40:51,721 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-12-11 22:40:51,721 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-11 22:40:51,722 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-12-11T22:40:51,737 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-12-11 22:40:51,740 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-11 22:40:51,741 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-11T22:40:51,958 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9983877b-ed49-7b76-d831-28cd314ed815 2024-12-11T22:40:51,973 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-11T22:40:51,987 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9983877b-ed49-7b76-d831-28cd314ed815/cluster_82fbc6ba-d42c-c9ee-739d-a2b1302b6f9a, deleteOnExit=true 2024-12-11T22:40:51,987 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-11T22:40:51,988 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9983877b-ed49-7b76-d831-28cd314ed815/test.cache.data in system properties and HBase conf 2024-12-11T22:40:51,988 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9983877b-ed49-7b76-d831-28cd314ed815/hadoop.tmp.dir in system properties and HBase conf 2024-12-11T22:40:51,989 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9983877b-ed49-7b76-d831-28cd314ed815/hadoop.log.dir in system properties and HBase conf 2024-12-11T22:40:51,989 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9983877b-ed49-7b76-d831-28cd314ed815/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-11T22:40:51,990 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9983877b-ed49-7b76-d831-28cd314ed815/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-11T22:40:51,990 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-11T22:40:52,065 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-11T22:40:52,148 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T22:40:52,151 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9983877b-ed49-7b76-d831-28cd314ed815/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-11T22:40:52,151 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9983877b-ed49-7b76-d831-28cd314ed815/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-11T22:40:52,152 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9983877b-ed49-7b76-d831-28cd314ed815/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-11T22:40:52,152 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9983877b-ed49-7b76-d831-28cd314ed815/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-11T22:40:52,153 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9983877b-ed49-7b76-d831-28cd314ed815/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-11T22:40:52,153 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9983877b-ed49-7b76-d831-28cd314ed815/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-11T22:40:52,153 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9983877b-ed49-7b76-d831-28cd314ed815/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-11T22:40:52,154 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9983877b-ed49-7b76-d831-28cd314ed815/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-11T22:40:52,154 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9983877b-ed49-7b76-d831-28cd314ed815/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-11T22:40:52,154 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9983877b-ed49-7b76-d831-28cd314ed815/nfs.dump.dir in system properties and HBase conf 2024-12-11T22:40:52,155 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9983877b-ed49-7b76-d831-28cd314ed815/java.io.tmpdir in system properties and HBase conf 2024-12-11T22:40:52,155 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9983877b-ed49-7b76-d831-28cd314ed815/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-11T22:40:52,155 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9983877b-ed49-7b76-d831-28cd314ed815/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-11T22:40:52,156 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9983877b-ed49-7b76-d831-28cd314ed815/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-11T22:40:53,087 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-11T22:40:53,150 INFO [Time-limited test {}] log.Log(170): Logging initialized @2187ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-11T22:40:53,211 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-11T22:40:53,264 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-11T22:40:53,281 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-11T22:40:53,281 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-11T22:40:53,282 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-11T22:40:53,294 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-11T22:40:53,297 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@265b24f1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9983877b-ed49-7b76-d831-28cd314ed815/hadoop.log.dir/,AVAILABLE} 2024-12-11T22:40:53,298 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@76beb053{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-11T22:40:53,458 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@376fa8f3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9983877b-ed49-7b76-d831-28cd314ed815/java.io.tmpdir/jetty-localhost-43525-hadoop-hdfs-3_4_1-tests_jar-_-any-15337757914103286131/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-11T22:40:53,464 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@672b88ea{HTTP/1.1, (http/1.1)}{localhost:43525} 2024-12-11T22:40:53,464 INFO [Time-limited test {}] server.Server(415): Started @2501ms 2024-12-11T22:40:53,922 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-11T22:40:53,930 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-11T22:40:53,931 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-11T22:40:53,931 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-11T22:40:53,932 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-11T22:40:53,932 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@739a104e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9983877b-ed49-7b76-d831-28cd314ed815/hadoop.log.dir/,AVAILABLE} 2024-12-11T22:40:53,933 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1c5d2af1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-11T22:40:54,034 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2226d6a6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9983877b-ed49-7b76-d831-28cd314ed815/java.io.tmpdir/jetty-localhost-33847-hadoop-hdfs-3_4_1-tests_jar-_-any-7572186039581437591/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-11T22:40:54,035 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1da1f773{HTTP/1.1, (http/1.1)}{localhost:33847} 2024-12-11T22:40:54,035 INFO [Time-limited test {}] server.Server(415): Started @3073ms 2024-12-11T22:40:54,086 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-11T22:40:54,188 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-11T22:40:54,196 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-11T22:40:54,197 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-11T22:40:54,197 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-11T22:40:54,197 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-11T22:40:54,198 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@34a3e1ff{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9983877b-ed49-7b76-d831-28cd314ed815/hadoop.log.dir/,AVAILABLE} 2024-12-11T22:40:54,199 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@12f16529{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-11T22:40:54,305 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7cfccf27{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9983877b-ed49-7b76-d831-28cd314ed815/java.io.tmpdir/jetty-localhost-41559-hadoop-hdfs-3_4_1-tests_jar-_-any-8702376836971274948/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-11T22:40:54,306 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6c41bbb7{HTTP/1.1, (http/1.1)}{localhost:41559} 2024-12-11T22:40:54,306 INFO [Time-limited test {}] server.Server(415): Started @3344ms 2024-12-11T22:40:54,308 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-11T22:40:54,343 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-11T22:40:54,347 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-11T22:40:54,351 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-11T22:40:54,351 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-11T22:40:54,351 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-11T22:40:54,353 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11ebaa12{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9983877b-ed49-7b76-d831-28cd314ed815/hadoop.log.dir/,AVAILABLE} 2024-12-11T22:40:54,354 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@52cf89e6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-11T22:40:54,448 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@67ddfd25{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9983877b-ed49-7b76-d831-28cd314ed815/java.io.tmpdir/jetty-localhost-44493-hadoop-hdfs-3_4_1-tests_jar-_-any-18443038527222713680/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-11T22:40:54,448 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@30e38263{HTTP/1.1, (http/1.1)}{localhost:44493} 2024-12-11T22:40:54,448 INFO [Time-limited test {}] server.Server(415): Started @3486ms 2024-12-11T22:40:54,450 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-11T22:40:55,239 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9983877b-ed49-7b76-d831-28cd314ed815/cluster_82fbc6ba-d42c-c9ee-739d-a2b1302b6f9a/data/data2/current/BP-2050408386-172.17.0.3-1733956852630/current, will proceed with Du for space computation calculation, 2024-12-11T22:40:55,239 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9983877b-ed49-7b76-d831-28cd314ed815/cluster_82fbc6ba-d42c-c9ee-739d-a2b1302b6f9a/data/data1/current/BP-2050408386-172.17.0.3-1733956852630/current, will proceed with Du for space computation calculation, 2024-12-11T22:40:55,239 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9983877b-ed49-7b76-d831-28cd314ed815/cluster_82fbc6ba-d42c-c9ee-739d-a2b1302b6f9a/data/data4/current/BP-2050408386-172.17.0.3-1733956852630/current, will proceed with Du for space computation calculation, 2024-12-11T22:40:55,239 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9983877b-ed49-7b76-d831-28cd314ed815/cluster_82fbc6ba-d42c-c9ee-739d-a2b1302b6f9a/data/data3/current/BP-2050408386-172.17.0.3-1733956852630/current, will proceed with Du for space computation calculation, 2024-12-11T22:40:55,276 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-11T22:40:55,277 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-11T22:40:55,322 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x63e4a1e915f31962 with lease ID 0xa999ad41c348d7a6: Processing first storage report for DS-f86c6cf2-47aa-4d2f-ba7f-902e5eaa2afe from datanode DatanodeRegistration(127.0.0.1:33033, datanodeUuid=9a67c07e-0e24-4ed3-956e-deaffa4c2ac4, infoPort=43991, infoSecurePort=0, ipcPort=34543, storageInfo=lv=-57;cid=testClusterID;nsid=1523418578;c=1733956852630) 2024-12-11T22:40:55,324 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x63e4a1e915f31962 with lease ID 0xa999ad41c348d7a6: from storage DS-f86c6cf2-47aa-4d2f-ba7f-902e5eaa2afe node DatanodeRegistration(127.0.0.1:33033, datanodeUuid=9a67c07e-0e24-4ed3-956e-deaffa4c2ac4, infoPort=43991, infoSecurePort=0, ipcPort=34543, storageInfo=lv=-57;cid=testClusterID;nsid=1523418578;c=1733956852630), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-11T22:40:55,324 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2f424988cd47c6e8 with lease ID 0xa999ad41c348d7a5: Processing first storage report for DS-34028945-25f0-40a0-8da9-6b2be7f53a5e from datanode DatanodeRegistration(127.0.0.1:42009, datanodeUuid=c0b77f01-0359-4346-88fd-f01272db9d4c, infoPort=46615, infoSecurePort=0, ipcPort=42755, storageInfo=lv=-57;cid=testClusterID;nsid=1523418578;c=1733956852630) 2024-12-11T22:40:55,325 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2f424988cd47c6e8 with lease ID 0xa999ad41c348d7a5: from storage DS-34028945-25f0-40a0-8da9-6b2be7f53a5e node DatanodeRegistration(127.0.0.1:42009, datanodeUuid=c0b77f01-0359-4346-88fd-f01272db9d4c, infoPort=46615, infoSecurePort=0, ipcPort=42755, storageInfo=lv=-57;cid=testClusterID;nsid=1523418578;c=1733956852630), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-11T22:40:55,325 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x63e4a1e915f31962 with lease ID 0xa999ad41c348d7a6: Processing first storage report for DS-e8a261f9-fa08-4800-b6cf-b15589f72547 from datanode DatanodeRegistration(127.0.0.1:33033, datanodeUuid=9a67c07e-0e24-4ed3-956e-deaffa4c2ac4, infoPort=43991, infoSecurePort=0, ipcPort=34543, storageInfo=lv=-57;cid=testClusterID;nsid=1523418578;c=1733956852630) 2024-12-11T22:40:55,325 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x63e4a1e915f31962 with lease ID 0xa999ad41c348d7a6: from storage DS-e8a261f9-fa08-4800-b6cf-b15589f72547 node DatanodeRegistration(127.0.0.1:33033, datanodeUuid=9a67c07e-0e24-4ed3-956e-deaffa4c2ac4, infoPort=43991, infoSecurePort=0, ipcPort=34543, storageInfo=lv=-57;cid=testClusterID;nsid=1523418578;c=1733956852630), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-11T22:40:55,325 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2f424988cd47c6e8 with lease ID 0xa999ad41c348d7a5: Processing first storage report for DS-c2080e4d-5fd9-4657-8e68-779c5208da5f from datanode DatanodeRegistration(127.0.0.1:42009, datanodeUuid=c0b77f01-0359-4346-88fd-f01272db9d4c, infoPort=46615, infoSecurePort=0, ipcPort=42755, storageInfo=lv=-57;cid=testClusterID;nsid=1523418578;c=1733956852630) 2024-12-11T22:40:55,326 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2f424988cd47c6e8 with lease ID 0xa999ad41c348d7a5: from storage DS-c2080e4d-5fd9-4657-8e68-779c5208da5f node DatanodeRegistration(127.0.0.1:42009, datanodeUuid=c0b77f01-0359-4346-88fd-f01272db9d4c, infoPort=46615, infoSecurePort=0, ipcPort=42755, storageInfo=lv=-57;cid=testClusterID;nsid=1523418578;c=1733956852630), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-11T22:40:55,341 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9983877b-ed49-7b76-d831-28cd314ed815/cluster_82fbc6ba-d42c-c9ee-739d-a2b1302b6f9a/data/data5/current/BP-2050408386-172.17.0.3-1733956852630/current, will proceed with Du for space computation calculation, 2024-12-11T22:40:55,341 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9983877b-ed49-7b76-d831-28cd314ed815/cluster_82fbc6ba-d42c-c9ee-739d-a2b1302b6f9a/data/data6/current/BP-2050408386-172.17.0.3-1733956852630/current, will proceed with Du for space computation calculation, 2024-12-11T22:40:55,362 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-11T22:40:55,366 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x20fa56daf477c3b with lease ID 0xa999ad41c348d7a7: Processing first storage report for DS-018fb24c-ac8f-4f8e-ace7-2b9f38ac5619 from datanode DatanodeRegistration(127.0.0.1:36811, datanodeUuid=e5facf27-141f-4985-85fa-81f8cc83574d, infoPort=44105, infoSecurePort=0, ipcPort=35277, storageInfo=lv=-57;cid=testClusterID;nsid=1523418578;c=1733956852630) 2024-12-11T22:40:55,366 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x20fa56daf477c3b with lease ID 0xa999ad41c348d7a7: from storage DS-018fb24c-ac8f-4f8e-ace7-2b9f38ac5619 node DatanodeRegistration(127.0.0.1:36811, datanodeUuid=e5facf27-141f-4985-85fa-81f8cc83574d, infoPort=44105, infoSecurePort=0, ipcPort=35277, storageInfo=lv=-57;cid=testClusterID;nsid=1523418578;c=1733956852630), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-11T22:40:55,367 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x20fa56daf477c3b with lease ID 0xa999ad41c348d7a7: Processing first storage report for DS-058c8b7f-a1a5-470c-ac46-9087a0dc21b1 from datanode DatanodeRegistration(127.0.0.1:36811, datanodeUuid=e5facf27-141f-4985-85fa-81f8cc83574d, infoPort=44105, infoSecurePort=0, ipcPort=35277, storageInfo=lv=-57;cid=testClusterID;nsid=1523418578;c=1733956852630) 2024-12-11T22:40:55,367 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x20fa56daf477c3b with lease ID 0xa999ad41c348d7a7: from storage DS-058c8b7f-a1a5-470c-ac46-9087a0dc21b1 node DatanodeRegistration(127.0.0.1:36811, datanodeUuid=e5facf27-141f-4985-85fa-81f8cc83574d, infoPort=44105, infoSecurePort=0, ipcPort=35277, storageInfo=lv=-57;cid=testClusterID;nsid=1523418578;c=1733956852630), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-11T22:40:55,444 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9983877b-ed49-7b76-d831-28cd314ed815 2024-12-11T22:40:55,504 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9983877b-ed49-7b76-d831-28cd314ed815/cluster_82fbc6ba-d42c-c9ee-739d-a2b1302b6f9a/zookeeper_0, clientPort=53529, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9983877b-ed49-7b76-d831-28cd314ed815/cluster_82fbc6ba-d42c-c9ee-739d-a2b1302b6f9a/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9983877b-ed49-7b76-d831-28cd314ed815/cluster_82fbc6ba-d42c-c9ee-739d-a2b1302b6f9a/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-11T22:40:55,512 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53529 2024-12-11T22:40:55,521 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T22:40:55,523 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T22:40:55,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741825_1001 (size=7) 2024-12-11T22:40:55,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741825_1001 (size=7) 2024-12-11T22:40:55,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741825_1001 (size=7) 2024-12-11T22:40:56,133 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c with version=8 2024-12-11T22:40:56,133 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/hbase-staging 2024-12-11T22:40:56,201 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-11T22:40:56,431 INFO [Time-limited test {}] client.ConnectionUtils(128): master/56d7a8405f4a:0 server-side Connection retries=45 2024-12-11T22:40:56,439 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T22:40:56,440 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-11T22:40:56,444 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-11T22:40:56,444 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T22:40:56,444 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-11T22:40:56,560 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-11T22:40:56,613 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-11T22:40:56,624 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-11T22:40:56,629 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-11T22:40:56,660 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 96009 (auto-detected) 2024-12-11T22:40:56,661 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-11T22:40:56,683 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:34843 2024-12-11T22:40:56,704 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34843 connecting to ZooKeeper ensemble=127.0.0.1:53529 2024-12-11T22:40:56,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:348430x0, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-11T22:40:56,828 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34843-0x1001717a55e0000 connected 2024-12-11T22:40:56,900 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T22:40:56,905 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T22:40:56,917 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T22:40:56,921 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c, hbase.cluster.distributed=false 2024-12-11T22:40:56,943 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-11T22:40:56,947 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34843 2024-12-11T22:40:56,948 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34843 2024-12-11T22:40:56,948 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34843 2024-12-11T22:40:56,949 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34843 2024-12-11T22:40:56,949 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34843 2024-12-11T22:40:57,045 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/56d7a8405f4a:0 server-side Connection retries=45 2024-12-11T22:40:57,047 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T22:40:57,047 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-11T22:40:57,047 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-11T22:40:57,048 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T22:40:57,048 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-11T22:40:57,051 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-11T22:40:57,052 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-11T22:40:57,053 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:34703 2024-12-11T22:40:57,055 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34703 connecting to ZooKeeper ensemble=127.0.0.1:53529 2024-12-11T22:40:57,056 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T22:40:57,058 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T22:40:57,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:347030x0, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-11T22:40:57,090 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34703-0x1001717a55e0001 connected 2024-12-11T22:40:57,090 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T22:40:57,094 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-11T22:40:57,102 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-11T22:40:57,105 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-11T22:40:57,110 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-11T22:40:57,111 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34703 2024-12-11T22:40:57,111 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34703 2024-12-11T22:40:57,112 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34703 2024-12-11T22:40:57,115 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34703 2024-12-11T22:40:57,116 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34703 2024-12-11T22:40:57,132 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/56d7a8405f4a:0 server-side Connection retries=45 2024-12-11T22:40:57,132 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T22:40:57,132 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-11T22:40:57,133 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-11T22:40:57,133 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T22:40:57,133 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-11T22:40:57,133 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-11T22:40:57,134 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-11T22:40:57,135 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40421 2024-12-11T22:40:57,137 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40421 connecting to ZooKeeper ensemble=127.0.0.1:53529 2024-12-11T22:40:57,138 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T22:40:57,140 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T22:40:57,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:404210x0, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-11T22:40:57,163 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40421-0x1001717a55e0002 connected 2024-12-11T22:40:57,163 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T22:40:57,164 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-11T22:40:57,167 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-11T22:40:57,168 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-11T22:40:57,171 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-11T22:40:57,171 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40421 2024-12-11T22:40:57,172 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40421 2024-12-11T22:40:57,172 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40421 2024-12-11T22:40:57,173 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40421 2024-12-11T22:40:57,173 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40421 2024-12-11T22:40:57,187 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/56d7a8405f4a:0 server-side Connection retries=45 2024-12-11T22:40:57,187 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T22:40:57,187 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-11T22:40:57,187 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-11T22:40:57,187 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T22:40:57,187 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-11T22:40:57,187 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-11T22:40:57,188 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-11T22:40:57,188 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36971 2024-12-11T22:40:57,190 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36971 connecting to ZooKeeper ensemble=127.0.0.1:53529 2024-12-11T22:40:57,191 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T22:40:57,192 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T22:40:57,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:369710x0, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-11T22:40:57,206 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:369710x0, quorum=127.0.0.1:53529, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T22:40:57,206 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36971-0x1001717a55e0003 connected 2024-12-11T22:40:57,207 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-11T22:40:57,208 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-11T22:40:57,209 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36971-0x1001717a55e0003, quorum=127.0.0.1:53529, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-11T22:40:57,212 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36971-0x1001717a55e0003, quorum=127.0.0.1:53529, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-11T22:40:57,213 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36971 2024-12-11T22:40:57,213 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36971 2024-12-11T22:40:57,214 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36971 2024-12-11T22:40:57,214 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36971 2024-12-11T22:40:57,215 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36971 2024-12-11T22:40:57,232 DEBUG [M:0;56d7a8405f4a:34843 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;56d7a8405f4a:34843 2024-12-11T22:40:57,233 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/56d7a8405f4a,34843,1733956856287 2024-12-11T22:40:57,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T22:40:57,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T22:40:57,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T22:40:57,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36971-0x1001717a55e0003, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T22:40:57,250 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/56d7a8405f4a,34843,1733956856287 2024-12-11T22:40:57,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36971-0x1001717a55e0003, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-11T22:40:57,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-11T22:40:57,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-11T22:40:57,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:40:57,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:40:57,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36971-0x1001717a55e0003, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:40:57,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:40:57,273 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-11T22:40:57,274 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/56d7a8405f4a,34843,1733956856287 from backup master directory 2024-12-11T22:40:57,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T22:40:57,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36971-0x1001717a55e0003, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T22:40:57,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T22:40:57,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/56d7a8405f4a,34843,1733956856287 2024-12-11T22:40:57,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T22:40:57,281 WARN [master/56d7a8405f4a:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-11T22:40:57,281 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=56d7a8405f4a,34843,1733956856287 2024-12-11T22:40:57,284 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-11T22:40:57,285 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-11T22:40:57,339 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/hbase.id] with ID: 520b6ef7-849a-4c82-8e75-768f6964b41e 2024-12-11T22:40:57,339 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/.tmp/hbase.id 2024-12-11T22:40:57,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741826_1002 (size=42) 2024-12-11T22:40:57,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741826_1002 (size=42) 2024-12-11T22:40:57,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741826_1002 (size=42) 2024-12-11T22:40:57,354 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/.tmp/hbase.id]:[hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/hbase.id] 2024-12-11T22:40:57,398 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T22:40:57,403 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-11T22:40:57,420 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-12-11T22:40:57,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:40:57,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:40:57,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:40:57,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36971-0x1001717a55e0003, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:40:57,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741827_1003 (size=196) 2024-12-11T22:40:57,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741827_1003 (size=196) 2024-12-11T22:40:57,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741827_1003 (size=196) 2024-12-11T22:40:57,484 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T22:40:57,486 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-11T22:40:57,499 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T22:40:57,502 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T22:40:57,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741828_1004 (size=1189) 2024-12-11T22:40:57,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741828_1004 (size=1189) 2024-12-11T22:40:57,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741828_1004 (size=1189) 2024-12-11T22:40:57,548 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/MasterData/data/master/store 2024-12-11T22:40:57,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741829_1005 (size=34) 2024-12-11T22:40:57,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741829_1005 (size=34) 2024-12-11T22:40:57,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741829_1005 (size=34) 2024-12-11T22:40:57,576 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-11T22:40:57,578 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:40:57,580 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-11T22:40:57,580 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T22:40:57,580 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T22:40:57,582 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-11T22:40:57,582 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T22:40:57,582 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T22:40:57,583 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733956857580Disabling compacts and flushes for region at 1733956857580Disabling writes for close at 1733956857582 (+2 ms)Writing region close event to WAL at 1733956857582Closed at 1733956857582 2024-12-11T22:40:57,585 WARN [master/56d7a8405f4a:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/MasterData/data/master/store/.initializing 2024-12-11T22:40:57,585 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/MasterData/WALs/56d7a8405f4a,34843,1733956856287 2024-12-11T22:40:57,592 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-11T22:40:57,606 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=56d7a8405f4a%2C34843%2C1733956856287, suffix=, logDir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/MasterData/WALs/56d7a8405f4a,34843,1733956856287, archiveDir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/MasterData/oldWALs, maxLogs=10 2024-12-11T22:40:57,626 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/MasterData/WALs/56d7a8405f4a,34843,1733956856287/56d7a8405f4a%2C34843%2C1733956856287.1733956857610, exclude list is [], retry=0 2024-12-11T22:40:57,642 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36811,DS-018fb24c-ac8f-4f8e-ace7-2b9f38ac5619,DISK] 2024-12-11T22:40:57,642 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33033,DS-f86c6cf2-47aa-4d2f-ba7f-902e5eaa2afe,DISK] 2024-12-11T22:40:57,642 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42009,DS-34028945-25f0-40a0-8da9-6b2be7f53a5e,DISK] 2024-12-11T22:40:57,644 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-11T22:40:57,680 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/MasterData/WALs/56d7a8405f4a,34843,1733956856287/56d7a8405f4a%2C34843%2C1733956856287.1733956857610 2024-12-11T22:40:57,681 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46615:46615),(127.0.0.1/127.0.0.1:44105:44105),(127.0.0.1/127.0.0.1:43991:43991)] 2024-12-11T22:40:57,682 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-11T22:40:57,682 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:40:57,685 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T22:40:57,686 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T22:40:57,718 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-11T22:40:57,738 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-11T22:40:57,740 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:40:57,743 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T22:40:57,743 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-11T22:40:57,746 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-11T22:40:57,746 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:40:57,747 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:40:57,747 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-11T22:40:57,749 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-11T22:40:57,750 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:40:57,750 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:40:57,751 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-11T22:40:57,753 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-11T22:40:57,753 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:40:57,754 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:40:57,754 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T22:40:57,758 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-11T22:40:57,759 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-11T22:40:57,764 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T22:40:57,765 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T22:40:57,768 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-11T22:40:57,772 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T22:40:57,776 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T22:40:57,777 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63496857, jitterRate=-0.05382309854030609}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-11T22:40:57,783 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733956857697Initializing all the Stores at 1733956857700 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733956857700Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733956857701 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733956857701Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733956857701Cleaning up temporary data from old regions at 1733956857765 (+64 ms)Region opened successfully at 1733956857783 (+18 ms) 2024-12-11T22:40:57,784 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-11T22:40:57,817 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dcdd82d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=56d7a8405f4a/172.17.0.3:0 2024-12-11T22:40:57,844 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-11T22:40:57,853 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-11T22:40:57,854 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-11T22:40:57,857 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-11T22:40:57,859 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-11T22:40:57,863 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-11T22:40:57,863 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-11T22:40:57,886 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-11T22:40:57,893 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-11T22:40:57,922 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-11T22:40:57,925 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-11T22:40:57,928 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-11T22:40:57,938 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-11T22:40:57,941 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-11T22:40:57,946 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-11T22:40:57,955 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-11T22:40:57,956 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-11T22:40:57,963 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-11T22:40:57,981 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-11T22:40:57,988 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-11T22:40:57,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-11T22:40:57,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-11T22:40:57,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-11T22:40:57,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36971-0x1001717a55e0003, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-11T22:40:57,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:40:57,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:40:57,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:40:57,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36971-0x1001717a55e0003, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:40:58,000 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=56d7a8405f4a,34843,1733956856287, sessionid=0x1001717a55e0000, setting cluster-up flag (Was=false) 2024-12-11T22:40:58,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:40:58,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:40:58,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:40:58,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36971-0x1001717a55e0003, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:40:58,055 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-11T22:40:58,060 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=56d7a8405f4a,34843,1733956856287 2024-12-11T22:40:58,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:40:58,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36971-0x1001717a55e0003, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:40:58,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:40:58,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:40:58,105 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-11T22:40:58,107 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=56d7a8405f4a,34843,1733956856287 2024-12-11T22:40:58,114 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-11T22:40:58,120 INFO [RS:1;56d7a8405f4a:40421 {}] regionserver.HRegionServer(746): ClusterId : 520b6ef7-849a-4c82-8e75-768f6964b41e 2024-12-11T22:40:58,120 INFO [RS:0;56d7a8405f4a:34703 {}] regionserver.HRegionServer(746): ClusterId : 520b6ef7-849a-4c82-8e75-768f6964b41e 2024-12-11T22:40:58,120 INFO [RS:2;56d7a8405f4a:36971 {}] regionserver.HRegionServer(746): ClusterId : 520b6ef7-849a-4c82-8e75-768f6964b41e 2024-12-11T22:40:58,123 DEBUG [RS:0;56d7a8405f4a:34703 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-11T22:40:58,123 DEBUG [RS:1;56d7a8405f4a:40421 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-11T22:40:58,123 DEBUG [RS:2;56d7a8405f4a:36971 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-11T22:40:58,132 DEBUG [RS:1;56d7a8405f4a:40421 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-11T22:40:58,132 DEBUG [RS:2;56d7a8405f4a:36971 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-11T22:40:58,132 DEBUG [RS:0;56d7a8405f4a:34703 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-11T22:40:58,132 DEBUG [RS:1;56d7a8405f4a:40421 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-11T22:40:58,132 DEBUG [RS:0;56d7a8405f4a:34703 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-11T22:40:58,132 DEBUG [RS:2;56d7a8405f4a:36971 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-11T22:40:58,139 DEBUG [RS:0;56d7a8405f4a:34703 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-11T22:40:58,139 DEBUG [RS:1;56d7a8405f4a:40421 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-11T22:40:58,139 DEBUG [RS:2;56d7a8405f4a:36971 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-11T22:40:58,140 DEBUG [RS:0;56d7a8405f4a:34703 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e5faceb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=56d7a8405f4a/172.17.0.3:0 2024-12-11T22:40:58,140 DEBUG [RS:1;56d7a8405f4a:40421 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55ea1c0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=56d7a8405f4a/172.17.0.3:0 2024-12-11T22:40:58,140 DEBUG [RS:2;56d7a8405f4a:36971 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b3fdce0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=56d7a8405f4a/172.17.0.3:0 2024-12-11T22:40:58,156 DEBUG [RS:0;56d7a8405f4a:34703 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;56d7a8405f4a:34703 2024-12-11T22:40:58,158 DEBUG [RS:2;56d7a8405f4a:36971 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;56d7a8405f4a:36971 2024-12-11T22:40:58,160 INFO [RS:2;56d7a8405f4a:36971 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-11T22:40:58,160 INFO [RS:0;56d7a8405f4a:34703 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-11T22:40:58,161 INFO [RS:2;56d7a8405f4a:36971 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-11T22:40:58,161 INFO [RS:0;56d7a8405f4a:34703 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-11T22:40:58,161 DEBUG [RS:2;56d7a8405f4a:36971 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-11T22:40:58,161 DEBUG [RS:0;56d7a8405f4a:34703 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-11T22:40:58,163 DEBUG [RS:1;56d7a8405f4a:40421 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;56d7a8405f4a:40421 2024-12-11T22:40:58,163 INFO [RS:1;56d7a8405f4a:40421 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-11T22:40:58,163 INFO [RS:1;56d7a8405f4a:40421 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-11T22:40:58,163 DEBUG [RS:1;56d7a8405f4a:40421 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-11T22:40:58,164 INFO [RS:0;56d7a8405f4a:34703 {}] regionserver.HRegionServer(2659): reportForDuty to master=56d7a8405f4a,34843,1733956856287 with port=34703, startcode=1733956857011 2024-12-11T22:40:58,164 INFO [RS:2;56d7a8405f4a:36971 {}] regionserver.HRegionServer(2659): reportForDuty to master=56d7a8405f4a,34843,1733956856287 with port=36971, startcode=1733956857186 2024-12-11T22:40:58,164 INFO [RS:1;56d7a8405f4a:40421 {}] regionserver.HRegionServer(2659): reportForDuty to master=56d7a8405f4a,34843,1733956856287 with port=40421, startcode=1733956857131 2024-12-11T22:40:58,177 DEBUG [RS:1;56d7a8405f4a:40421 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-11T22:40:58,177 DEBUG [RS:0;56d7a8405f4a:34703 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-11T22:40:58,177 DEBUG [RS:2;56d7a8405f4a:36971 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-11T22:40:58,210 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51809, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-11T22:40:58,210 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44387, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-11T22:40:58,210 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48025, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-11T22:40:58,212 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-11T22:40:58,216 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34843 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:40:58,220 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34843 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:40:58,221 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34843 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:40:58,224 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-11T22:40:58,232 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-11T22:40:58,240 DEBUG [RS:2;56d7a8405f4a:36971 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-11T22:40:58,240 DEBUG [RS:0;56d7a8405f4a:34703 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-11T22:40:58,240 DEBUG [RS:1;56d7a8405f4a:40421 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-11T22:40:58,240 WARN [RS:2;56d7a8405f4a:36971 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-11T22:40:58,240 WARN [RS:0;56d7a8405f4a:34703 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-11T22:40:58,240 WARN [RS:1;56d7a8405f4a:40421 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-11T22:40:58,239 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 56d7a8405f4a,34843,1733956856287 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-11T22:40:58,248 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/56d7a8405f4a:0, corePoolSize=5, maxPoolSize=5 2024-12-11T22:40:58,248 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/56d7a8405f4a:0, corePoolSize=5, maxPoolSize=5 2024-12-11T22:40:58,248 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/56d7a8405f4a:0, corePoolSize=5, maxPoolSize=5 2024-12-11T22:40:58,248 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/56d7a8405f4a:0, corePoolSize=5, maxPoolSize=5 2024-12-11T22:40:58,248 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/56d7a8405f4a:0, corePoolSize=10, maxPoolSize=10 2024-12-11T22:40:58,248 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/56d7a8405f4a:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:40:58,249 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/56d7a8405f4a:0, corePoolSize=2, maxPoolSize=2 2024-12-11T22:40:58,249 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/56d7a8405f4a:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:40:58,249 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733956888249 2024-12-11T22:40:58,251 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-11T22:40:58,252 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-11T22:40:58,254 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-11T22:40:58,254 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-11T22:40:58,255 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-11T22:40:58,255 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-11T22:40:58,256 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-11T22:40:58,256 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-11T22:40:58,256 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-11T22:40:58,261 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-11T22:40:58,261 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:40:58,262 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-11T22:40:58,262 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-11T22:40:58,262 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-11T22:40:58,264 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-11T22:40:58,264 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-11T22:40:58,266 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/56d7a8405f4a:0:becomeActiveMaster-HFileCleaner.large.0-1733956858266,5,FailOnTimeoutGroup] 2024-12-11T22:40:58,267 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/56d7a8405f4a:0:becomeActiveMaster-HFileCleaner.small.0-1733956858266,5,FailOnTimeoutGroup] 2024-12-11T22:40:58,267 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-11T22:40:58,267 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-11T22:40:58,268 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-11T22:40:58,268 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-11T22:40:58,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741831_1007 (size=1321) 2024-12-11T22:40:58,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741831_1007 (size=1321) 2024-12-11T22:40:58,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741831_1007 (size=1321) 2024-12-11T22:40:58,312 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-11T22:40:58,313 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c 2024-12-11T22:40:58,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741832_1008 (size=32) 2024-12-11T22:40:58,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741832_1008 (size=32) 2024-12-11T22:40:58,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741832_1008 (size=32) 2024-12-11T22:40:58,328 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:40:58,331 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-11T22:40:58,333 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-11T22:40:58,334 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:40:58,334 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T22:40:58,334 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-11T22:40:58,337 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-11T22:40:58,337 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:40:58,338 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T22:40:58,338 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-11T22:40:58,340 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-11T22:40:58,340 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:40:58,341 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T22:40:58,341 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-11T22:40:58,341 INFO [RS:0;56d7a8405f4a:34703 {}] regionserver.HRegionServer(2659): reportForDuty to master=56d7a8405f4a,34843,1733956856287 with port=34703, startcode=1733956857011 2024-12-11T22:40:58,341 INFO [RS:1;56d7a8405f4a:40421 {}] regionserver.HRegionServer(2659): reportForDuty to master=56d7a8405f4a,34843,1733956856287 with port=40421, startcode=1733956857131 2024-12-11T22:40:58,341 INFO [RS:2;56d7a8405f4a:36971 {}] regionserver.HRegionServer(2659): reportForDuty to master=56d7a8405f4a,34843,1733956856287 with port=36971, startcode=1733956857186 2024-12-11T22:40:58,343 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34843 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 56d7a8405f4a,34703,1733956857011 2024-12-11T22:40:58,344 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-11T22:40:58,344 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:40:58,345 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T22:40:58,345 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-11T22:40:58,346 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34843 {}] master.ServerManager(517): Registering regionserver=56d7a8405f4a,34703,1733956857011 2024-12-11T22:40:58,346 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/hbase/meta/1588230740 2024-12-11T22:40:58,347 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/hbase/meta/1588230740 2024-12-11T22:40:58,351 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-11T22:40:58,351 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-11T22:40:58,352 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-11T22:40:58,354 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-11T22:40:58,354 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34843 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 56d7a8405f4a,36971,1733956857186 2024-12-11T22:40:58,355 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34843 {}] master.ServerManager(517): Registering regionserver=56d7a8405f4a,36971,1733956857186 2024-12-11T22:40:58,355 DEBUG [RS:0;56d7a8405f4a:34703 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c 2024-12-11T22:40:58,355 DEBUG [RS:0;56d7a8405f4a:34703 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43559 2024-12-11T22:40:58,355 DEBUG [RS:0;56d7a8405f4a:34703 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-11T22:40:58,359 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T22:40:58,359 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34843 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 56d7a8405f4a,40421,1733956857131 2024-12-11T22:40:58,359 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34843 {}] master.ServerManager(517): Registering regionserver=56d7a8405f4a,40421,1733956857131 2024-12-11T22:40:58,360 DEBUG [RS:2;56d7a8405f4a:36971 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c 2024-12-11T22:40:58,360 DEBUG [RS:2;56d7a8405f4a:36971 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43559 2024-12-11T22:40:58,360 DEBUG [RS:2;56d7a8405f4a:36971 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-11T22:40:58,360 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61815205, jitterRate=-0.07888166606426239}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-11T22:40:58,363 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733956858328Initializing all the Stores at 1733956858330 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733956858330Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733956858330Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733956858331 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733956858331Cleaning up temporary data from old regions at 1733956858351 (+20 ms)Region opened successfully at 1733956858363 (+12 ms) 2024-12-11T22:40:58,364 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-11T22:40:58,364 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-11T22:40:58,364 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-11T22:40:58,364 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-11T22:40:58,364 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-11T22:40:58,364 DEBUG [RS:1;56d7a8405f4a:40421 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c 2024-12-11T22:40:58,365 DEBUG [RS:1;56d7a8405f4a:40421 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43559 2024-12-11T22:40:58,365 DEBUG [RS:1;56d7a8405f4a:40421 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-11T22:40:58,365 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-11T22:40:58,365 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733956858364Disabling compacts and flushes for region at 1733956858364Disabling writes for close at 1733956858364Writing region close event to WAL at 1733956858365 (+1 ms)Closed at 1733956858365 2024-12-11T22:40:58,368 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-11T22:40:58,368 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-11T22:40:58,373 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-11T22:40:58,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-11T22:40:58,383 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-11T22:40:58,385 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-11T22:40:58,418 DEBUG [RS:0;56d7a8405f4a:34703 {}] zookeeper.ZKUtil(111): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/56d7a8405f4a,34703,1733956857011 2024-12-11T22:40:58,418 WARN [RS:0;56d7a8405f4a:34703 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-11T22:40:58,418 INFO [RS:0;56d7a8405f4a:34703 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T22:40:58,418 DEBUG [RS:2;56d7a8405f4a:36971 {}] zookeeper.ZKUtil(111): regionserver:36971-0x1001717a55e0003, quorum=127.0.0.1:53529, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/56d7a8405f4a,36971,1733956857186 2024-12-11T22:40:58,418 DEBUG [RS:1;56d7a8405f4a:40421 {}] zookeeper.ZKUtil(111): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/56d7a8405f4a,40421,1733956857131 2024-12-11T22:40:58,418 DEBUG [RS:0;56d7a8405f4a:34703 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,34703,1733956857011 2024-12-11T22:40:58,418 WARN [RS:2;56d7a8405f4a:36971 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-11T22:40:58,418 WARN [RS:1;56d7a8405f4a:40421 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-11T22:40:58,419 INFO [RS:2;56d7a8405f4a:36971 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T22:40:58,419 INFO [RS:1;56d7a8405f4a:40421 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T22:40:58,419 DEBUG [RS:2;56d7a8405f4a:36971 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,36971,1733956857186 2024-12-11T22:40:58,419 DEBUG [RS:1;56d7a8405f4a:40421 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,40421,1733956857131 2024-12-11T22:40:58,420 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [56d7a8405f4a,40421,1733956857131] 2024-12-11T22:40:58,420 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [56d7a8405f4a,34703,1733956857011] 2024-12-11T22:40:58,421 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [56d7a8405f4a,36971,1733956857186] 2024-12-11T22:40:58,447 INFO [RS:0;56d7a8405f4a:34703 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-11T22:40:58,447 INFO [RS:1;56d7a8405f4a:40421 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-11T22:40:58,447 INFO [RS:2;56d7a8405f4a:36971 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-11T22:40:58,461 INFO [RS:0;56d7a8405f4a:34703 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-11T22:40:58,461 INFO [RS:1;56d7a8405f4a:40421 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-11T22:40:58,461 INFO [RS:2;56d7a8405f4a:36971 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-11T22:40:58,466 INFO [RS:2;56d7a8405f4a:36971 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-11T22:40:58,466 INFO [RS:1;56d7a8405f4a:40421 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-11T22:40:58,466 INFO [RS:0;56d7a8405f4a:34703 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-11T22:40:58,466 INFO [RS:2;56d7a8405f4a:36971 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T22:40:58,466 INFO [RS:0;56d7a8405f4a:34703 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T22:40:58,466 INFO [RS:1;56d7a8405f4a:40421 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T22:40:58,467 INFO [RS:0;56d7a8405f4a:34703 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-11T22:40:58,467 INFO [RS:1;56d7a8405f4a:40421 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-11T22:40:58,467 INFO [RS:2;56d7a8405f4a:36971 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-11T22:40:58,472 INFO [RS:0;56d7a8405f4a:34703 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-11T22:40:58,472 INFO [RS:1;56d7a8405f4a:40421 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-11T22:40:58,472 INFO [RS:2;56d7a8405f4a:36971 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-11T22:40:58,473 INFO [RS:0;56d7a8405f4a:34703 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-11T22:40:58,473 INFO [RS:1;56d7a8405f4a:40421 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-11T22:40:58,473 INFO [RS:2;56d7a8405f4a:36971 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-11T22:40:58,474 DEBUG [RS:0;56d7a8405f4a:34703 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/56d7a8405f4a:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:40:58,474 DEBUG [RS:2;56d7a8405f4a:36971 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/56d7a8405f4a:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:40:58,474 DEBUG [RS:1;56d7a8405f4a:40421 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/56d7a8405f4a:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:40:58,474 DEBUG [RS:0;56d7a8405f4a:34703 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/56d7a8405f4a:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:40:58,474 DEBUG [RS:1;56d7a8405f4a:40421 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/56d7a8405f4a:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:40:58,474 DEBUG [RS:2;56d7a8405f4a:36971 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/56d7a8405f4a:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:40:58,474 DEBUG [RS:0;56d7a8405f4a:34703 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/56d7a8405f4a:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:40:58,474 DEBUG [RS:1;56d7a8405f4a:40421 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/56d7a8405f4a:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:40:58,474 DEBUG [RS:0;56d7a8405f4a:34703 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/56d7a8405f4a:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:40:58,474 DEBUG [RS:2;56d7a8405f4a:36971 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/56d7a8405f4a:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:40:58,474 DEBUG [RS:0;56d7a8405f4a:34703 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/56d7a8405f4a:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:40:58,474 DEBUG [RS:1;56d7a8405f4a:40421 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/56d7a8405f4a:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:40:58,474 DEBUG [RS:2;56d7a8405f4a:36971 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/56d7a8405f4a:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:40:58,474 DEBUG [RS:0;56d7a8405f4a:34703 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/56d7a8405f4a:0, corePoolSize=2, maxPoolSize=2 2024-12-11T22:40:58,474 DEBUG [RS:1;56d7a8405f4a:40421 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/56d7a8405f4a:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:40:58,474 DEBUG [RS:2;56d7a8405f4a:36971 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/56d7a8405f4a:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:40:58,474 DEBUG [RS:0;56d7a8405f4a:34703 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/56d7a8405f4a:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:40:58,474 DEBUG [RS:1;56d7a8405f4a:40421 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/56d7a8405f4a:0, corePoolSize=2, maxPoolSize=2 2024-12-11T22:40:58,474 DEBUG [RS:0;56d7a8405f4a:34703 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/56d7a8405f4a:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:40:58,474 DEBUG [RS:2;56d7a8405f4a:36971 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/56d7a8405f4a:0, corePoolSize=2, maxPoolSize=2 2024-12-11T22:40:58,475 DEBUG [RS:1;56d7a8405f4a:40421 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/56d7a8405f4a:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:40:58,475 DEBUG [RS:0;56d7a8405f4a:34703 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/56d7a8405f4a:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:40:58,475 DEBUG [RS:2;56d7a8405f4a:36971 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/56d7a8405f4a:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:40:58,475 DEBUG [RS:0;56d7a8405f4a:34703 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/56d7a8405f4a:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:40:58,475 DEBUG [RS:1;56d7a8405f4a:40421 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/56d7a8405f4a:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:40:58,475 DEBUG [RS:2;56d7a8405f4a:36971 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/56d7a8405f4a:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:40:58,475 DEBUG [RS:0;56d7a8405f4a:34703 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/56d7a8405f4a:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:40:58,475 DEBUG [RS:1;56d7a8405f4a:40421 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/56d7a8405f4a:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:40:58,475 DEBUG [RS:0;56d7a8405f4a:34703 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/56d7a8405f4a:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:40:58,475 DEBUG [RS:2;56d7a8405f4a:36971 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/56d7a8405f4a:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:40:58,475 DEBUG [RS:0;56d7a8405f4a:34703 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/56d7a8405f4a:0, corePoolSize=3, maxPoolSize=3 2024-12-11T22:40:58,475 DEBUG [RS:1;56d7a8405f4a:40421 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/56d7a8405f4a:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:40:58,475 DEBUG [RS:2;56d7a8405f4a:36971 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/56d7a8405f4a:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:40:58,475 DEBUG [RS:0;56d7a8405f4a:34703 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/56d7a8405f4a:0, corePoolSize=3, maxPoolSize=3 2024-12-11T22:40:58,475 DEBUG [RS:1;56d7a8405f4a:40421 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/56d7a8405f4a:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:40:58,475 DEBUG [RS:2;56d7a8405f4a:36971 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/56d7a8405f4a:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:40:58,475 DEBUG [RS:2;56d7a8405f4a:36971 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/56d7a8405f4a:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:40:58,475 DEBUG [RS:1;56d7a8405f4a:40421 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/56d7a8405f4a:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:40:58,475 DEBUG [RS:2;56d7a8405f4a:36971 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/56d7a8405f4a:0, corePoolSize=3, maxPoolSize=3 2024-12-11T22:40:58,475 DEBUG [RS:1;56d7a8405f4a:40421 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/56d7a8405f4a:0, corePoolSize=3, maxPoolSize=3 2024-12-11T22:40:58,476 DEBUG [RS:1;56d7a8405f4a:40421 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/56d7a8405f4a:0, corePoolSize=3, maxPoolSize=3 2024-12-11T22:40:58,476 DEBUG [RS:2;56d7a8405f4a:36971 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/56d7a8405f4a:0, corePoolSize=3, maxPoolSize=3 2024-12-11T22:40:58,477 INFO [RS:2;56d7a8405f4a:36971 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-11T22:40:58,477 INFO [RS:1;56d7a8405f4a:40421 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-11T22:40:58,477 INFO [RS:2;56d7a8405f4a:36971 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-11T22:40:58,477 INFO [RS:0;56d7a8405f4a:34703 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-11T22:40:58,477 INFO [RS:1;56d7a8405f4a:40421 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-11T22:40:58,477 INFO [RS:2;56d7a8405f4a:36971 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T22:40:58,477 INFO [RS:0;56d7a8405f4a:34703 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-11T22:40:58,477 INFO [RS:1;56d7a8405f4a:40421 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T22:40:58,477 INFO [RS:0;56d7a8405f4a:34703 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T22:40:58,477 INFO [RS:2;56d7a8405f4a:36971 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-11T22:40:58,477 INFO [RS:1;56d7a8405f4a:40421 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-11T22:40:58,477 INFO [RS:0;56d7a8405f4a:34703 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-11T22:40:58,477 INFO [RS:2;56d7a8405f4a:36971 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-11T22:40:58,477 INFO [RS:1;56d7a8405f4a:40421 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-11T22:40:58,477 INFO [RS:0;56d7a8405f4a:34703 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-11T22:40:58,477 INFO [RS:2;56d7a8405f4a:36971 {}] hbase.ChoreService(168): Chore ScheduledChore name=56d7a8405f4a,36971,1733956857186-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-11T22:40:58,477 INFO [RS:1;56d7a8405f4a:40421 {}] hbase.ChoreService(168): Chore ScheduledChore name=56d7a8405f4a,40421,1733956857131-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-11T22:40:58,477 INFO [RS:0;56d7a8405f4a:34703 {}] hbase.ChoreService(168): Chore ScheduledChore name=56d7a8405f4a,34703,1733956857011-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-11T22:40:58,493 INFO [RS:2;56d7a8405f4a:36971 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-11T22:40:58,495 INFO [RS:2;56d7a8405f4a:36971 {}] hbase.ChoreService(168): Chore ScheduledChore name=56d7a8405f4a,36971,1733956857186-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T22:40:58,495 INFO [RS:2;56d7a8405f4a:36971 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T22:40:58,495 INFO [RS:2;56d7a8405f4a:36971 {}] regionserver.Replication(171): 56d7a8405f4a,36971,1733956857186 started 2024-12-11T22:40:58,498 INFO [RS:1;56d7a8405f4a:40421 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-11T22:40:58,498 INFO [RS:0;56d7a8405f4a:34703 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-11T22:40:58,498 INFO [RS:1;56d7a8405f4a:40421 {}] hbase.ChoreService(168): Chore ScheduledChore name=56d7a8405f4a,40421,1733956857131-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T22:40:58,498 INFO [RS:0;56d7a8405f4a:34703 {}] hbase.ChoreService(168): Chore ScheduledChore name=56d7a8405f4a,34703,1733956857011-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T22:40:58,499 INFO [RS:1;56d7a8405f4a:40421 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T22:40:58,499 INFO [RS:0;56d7a8405f4a:34703 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T22:40:58,499 INFO [RS:1;56d7a8405f4a:40421 {}] regionserver.Replication(171): 56d7a8405f4a,40421,1733956857131 started 2024-12-11T22:40:58,499 INFO [RS:0;56d7a8405f4a:34703 {}] regionserver.Replication(171): 56d7a8405f4a,34703,1733956857011 started 2024-12-11T22:40:58,511 INFO [RS:2;56d7a8405f4a:36971 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T22:40:58,511 INFO [RS:2;56d7a8405f4a:36971 {}] regionserver.HRegionServer(1482): Serving as 56d7a8405f4a,36971,1733956857186, RpcServer on 56d7a8405f4a/172.17.0.3:36971, sessionid=0x1001717a55e0003 2024-12-11T22:40:58,512 DEBUG [RS:2;56d7a8405f4a:36971 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-11T22:40:58,512 DEBUG [RS:2;56d7a8405f4a:36971 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 56d7a8405f4a,36971,1733956857186 2024-12-11T22:40:58,512 DEBUG [RS:2;56d7a8405f4a:36971 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '56d7a8405f4a,36971,1733956857186' 2024-12-11T22:40:58,512 DEBUG [RS:2;56d7a8405f4a:36971 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-11T22:40:58,513 DEBUG [RS:2;56d7a8405f4a:36971 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-11T22:40:58,513 DEBUG [RS:2;56d7a8405f4a:36971 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-11T22:40:58,513 DEBUG [RS:2;56d7a8405f4a:36971 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-11T22:40:58,514 DEBUG [RS:2;56d7a8405f4a:36971 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 56d7a8405f4a,36971,1733956857186 2024-12-11T22:40:58,514 DEBUG [RS:2;56d7a8405f4a:36971 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '56d7a8405f4a,36971,1733956857186' 2024-12-11T22:40:58,514 DEBUG [RS:2;56d7a8405f4a:36971 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-11T22:40:58,514 DEBUG [RS:2;56d7a8405f4a:36971 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-11T22:40:58,515 DEBUG [RS:2;56d7a8405f4a:36971 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-11T22:40:58,515 INFO [RS:2;56d7a8405f4a:36971 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-11T22:40:58,515 INFO [RS:2;56d7a8405f4a:36971 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-11T22:40:58,520 INFO [RS:0;56d7a8405f4a:34703 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T22:40:58,520 INFO [RS:1;56d7a8405f4a:40421 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T22:40:58,520 INFO [RS:0;56d7a8405f4a:34703 {}] regionserver.HRegionServer(1482): Serving as 56d7a8405f4a,34703,1733956857011, RpcServer on 56d7a8405f4a/172.17.0.3:34703, sessionid=0x1001717a55e0001 2024-12-11T22:40:58,520 INFO [RS:1;56d7a8405f4a:40421 {}] regionserver.HRegionServer(1482): Serving as 56d7a8405f4a,40421,1733956857131, RpcServer on 56d7a8405f4a/172.17.0.3:40421, sessionid=0x1001717a55e0002 2024-12-11T22:40:58,520 DEBUG [RS:1;56d7a8405f4a:40421 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-11T22:40:58,520 DEBUG [RS:0;56d7a8405f4a:34703 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-11T22:40:58,520 DEBUG [RS:0;56d7a8405f4a:34703 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 56d7a8405f4a,34703,1733956857011 2024-12-11T22:40:58,520 DEBUG [RS:1;56d7a8405f4a:40421 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 56d7a8405f4a,40421,1733956857131 2024-12-11T22:40:58,520 DEBUG [RS:1;56d7a8405f4a:40421 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '56d7a8405f4a,40421,1733956857131' 2024-12-11T22:40:58,520 DEBUG [RS:0;56d7a8405f4a:34703 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '56d7a8405f4a,34703,1733956857011' 2024-12-11T22:40:58,520 DEBUG [RS:1;56d7a8405f4a:40421 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-11T22:40:58,520 DEBUG [RS:0;56d7a8405f4a:34703 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-11T22:40:58,521 DEBUG [RS:1;56d7a8405f4a:40421 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-11T22:40:58,521 DEBUG [RS:0;56d7a8405f4a:34703 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-11T22:40:58,522 DEBUG [RS:1;56d7a8405f4a:40421 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-11T22:40:58,522 DEBUG [RS:0;56d7a8405f4a:34703 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-11T22:40:58,522 DEBUG [RS:0;56d7a8405f4a:34703 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-11T22:40:58,522 DEBUG [RS:1;56d7a8405f4a:40421 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-11T22:40:58,522 DEBUG [RS:0;56d7a8405f4a:34703 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 56d7a8405f4a,34703,1733956857011 2024-12-11T22:40:58,522 DEBUG [RS:1;56d7a8405f4a:40421 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 56d7a8405f4a,40421,1733956857131 2024-12-11T22:40:58,522 DEBUG [RS:1;56d7a8405f4a:40421 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '56d7a8405f4a,40421,1733956857131' 2024-12-11T22:40:58,522 DEBUG [RS:0;56d7a8405f4a:34703 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '56d7a8405f4a,34703,1733956857011' 2024-12-11T22:40:58,522 DEBUG [RS:1;56d7a8405f4a:40421 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-11T22:40:58,522 DEBUG [RS:0;56d7a8405f4a:34703 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-11T22:40:58,523 DEBUG [RS:0;56d7a8405f4a:34703 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-11T22:40:58,523 DEBUG [RS:1;56d7a8405f4a:40421 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-11T22:40:58,523 DEBUG [RS:0;56d7a8405f4a:34703 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-11T22:40:58,523 DEBUG [RS:1;56d7a8405f4a:40421 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-11T22:40:58,523 INFO [RS:0;56d7a8405f4a:34703 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-11T22:40:58,523 INFO [RS:1;56d7a8405f4a:40421 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-11T22:40:58,523 INFO [RS:0;56d7a8405f4a:34703 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-11T22:40:58,523 INFO [RS:1;56d7a8405f4a:40421 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-11T22:40:58,536 WARN [56d7a8405f4a:34843 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T22:40:58,623 INFO [RS:2;56d7a8405f4a:36971 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-11T22:40:58,624 INFO [RS:1;56d7a8405f4a:40421 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-11T22:40:58,624 INFO [RS:0;56d7a8405f4a:34703 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-11T22:40:58,628 INFO [RS:2;56d7a8405f4a:36971 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=56d7a8405f4a%2C36971%2C1733956857186, suffix=, logDir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,36971,1733956857186, archiveDir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/oldWALs, maxLogs=32 2024-12-11T22:40:58,629 INFO [RS:1;56d7a8405f4a:40421 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=56d7a8405f4a%2C40421%2C1733956857131, suffix=, logDir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,40421,1733956857131, archiveDir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/oldWALs, maxLogs=32 2024-12-11T22:40:58,629 INFO [RS:0;56d7a8405f4a:34703 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=56d7a8405f4a%2C34703%2C1733956857011, suffix=, logDir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,34703,1733956857011, archiveDir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/oldWALs, maxLogs=32 2024-12-11T22:40:58,643 DEBUG [RS:1;56d7a8405f4a:40421 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,40421,1733956857131/56d7a8405f4a%2C40421%2C1733956857131.1733956858632, exclude list is [], retry=0 2024-12-11T22:40:58,645 DEBUG [RS:2;56d7a8405f4a:36971 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,36971,1733956857186/56d7a8405f4a%2C36971%2C1733956857186.1733956858632, exclude list is [], retry=0 2024-12-11T22:40:58,645 DEBUG [RS:0;56d7a8405f4a:34703 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,34703,1733956857011/56d7a8405f4a%2C34703%2C1733956857011.1733956858632, exclude list is [], retry=0 2024-12-11T22:40:58,648 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42009,DS-34028945-25f0-40a0-8da9-6b2be7f53a5e,DISK] 2024-12-11T22:40:58,648 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33033,DS-f86c6cf2-47aa-4d2f-ba7f-902e5eaa2afe,DISK] 2024-12-11T22:40:58,648 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42009,DS-34028945-25f0-40a0-8da9-6b2be7f53a5e,DISK] 2024-12-11T22:40:58,649 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33033,DS-f86c6cf2-47aa-4d2f-ba7f-902e5eaa2afe,DISK] 2024-12-11T22:40:58,649 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36811,DS-018fb24c-ac8f-4f8e-ace7-2b9f38ac5619,DISK] 2024-12-11T22:40:58,650 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33033,DS-f86c6cf2-47aa-4d2f-ba7f-902e5eaa2afe,DISK] 2024-12-11T22:40:58,650 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36811,DS-018fb24c-ac8f-4f8e-ace7-2b9f38ac5619,DISK] 2024-12-11T22:40:58,650 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36811,DS-018fb24c-ac8f-4f8e-ace7-2b9f38ac5619,DISK] 2024-12-11T22:40:58,650 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42009,DS-34028945-25f0-40a0-8da9-6b2be7f53a5e,DISK] 2024-12-11T22:40:58,654 INFO [RS:0;56d7a8405f4a:34703 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,34703,1733956857011/56d7a8405f4a%2C34703%2C1733956857011.1733956858632 2024-12-11T22:40:58,654 INFO [RS:2;56d7a8405f4a:36971 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,36971,1733956857186/56d7a8405f4a%2C36971%2C1733956857186.1733956858632 2024-12-11T22:40:58,654 INFO [RS:1;56d7a8405f4a:40421 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,40421,1733956857131/56d7a8405f4a%2C40421%2C1733956857131.1733956858632 2024-12-11T22:40:58,655 DEBUG [RS:0;56d7a8405f4a:34703 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46615:46615),(127.0.0.1/127.0.0.1:44105:44105),(127.0.0.1/127.0.0.1:43991:43991)] 2024-12-11T22:40:58,655 DEBUG [RS:2;56d7a8405f4a:36971 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43991:43991),(127.0.0.1/127.0.0.1:44105:44105),(127.0.0.1/127.0.0.1:46615:46615)] 2024-12-11T22:40:58,655 DEBUG [RS:1;56d7a8405f4a:40421 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43991:43991),(127.0.0.1/127.0.0.1:46615:46615),(127.0.0.1/127.0.0.1:44105:44105)] 2024-12-11T22:40:58,789 DEBUG [56d7a8405f4a:34843 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-11T22:40:58,797 DEBUG [56d7a8405f4a:34843 {}] balancer.BalancerClusterState(204): Hosts are {56d7a8405f4a=0} racks are {/default-rack=0} 2024-12-11T22:40:58,802 DEBUG [56d7a8405f4a:34843 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-11T22:40:58,802 DEBUG [56d7a8405f4a:34843 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-11T22:40:58,802 DEBUG [56d7a8405f4a:34843 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-11T22:40:58,802 DEBUG [56d7a8405f4a:34843 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-11T22:40:58,802 DEBUG [56d7a8405f4a:34843 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-11T22:40:58,802 DEBUG [56d7a8405f4a:34843 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-11T22:40:58,803 INFO [56d7a8405f4a:34843 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-11T22:40:58,803 INFO [56d7a8405f4a:34843 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-11T22:40:58,803 INFO [56d7a8405f4a:34843 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-11T22:40:58,803 DEBUG [56d7a8405f4a:34843 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-11T22:40:58,809 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=56d7a8405f4a,36971,1733956857186 2024-12-11T22:40:58,815 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 56d7a8405f4a,36971,1733956857186, state=OPENING 2024-12-11T22:40:58,855 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-11T22:40:58,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:40:58,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:40:58,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:40:58,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36971-0x1001717a55e0003, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:40:58,907 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:40:58,907 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:40:58,907 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:40:58,907 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:40:58,909 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-11T22:40:58,913 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,36971,1733956857186}] 2024-12-11T22:40:59,093 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-11T22:40:59,095 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47987, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-11T22:40:59,106 INFO [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-11T22:40:59,106 INFO [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T22:40:59,107 INFO [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-11T22:40:59,110 INFO [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=56d7a8405f4a%2C36971%2C1733956857186.meta, suffix=.meta, logDir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,36971,1733956857186, archiveDir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/oldWALs, maxLogs=32 2024-12-11T22:40:59,124 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,36971,1733956857186/56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, exclude list is [], retry=0 2024-12-11T22:40:59,128 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36811,DS-018fb24c-ac8f-4f8e-ace7-2b9f38ac5619,DISK] 2024-12-11T22:40:59,128 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33033,DS-f86c6cf2-47aa-4d2f-ba7f-902e5eaa2afe,DISK] 2024-12-11T22:40:59,128 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42009,DS-34028945-25f0-40a0-8da9-6b2be7f53a5e,DISK] 2024-12-11T22:40:59,131 INFO [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,36971,1733956857186/56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta 2024-12-11T22:40:59,131 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46615:46615),(127.0.0.1/127.0.0.1:44105:44105),(127.0.0.1/127.0.0.1:43991:43991)] 2024-12-11T22:40:59,132 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-11T22:40:59,133 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-11T22:40:59,136 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-11T22:40:59,140 INFO [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-11T22:40:59,144 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-11T22:40:59,144 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:40:59,144 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-11T22:40:59,144 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-11T22:40:59,148 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-11T22:40:59,149 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-11T22:40:59,149 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:40:59,150 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T22:40:59,150 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-11T22:40:59,151 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-11T22:40:59,151 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:40:59,152 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T22:40:59,152 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-11T22:40:59,154 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-11T22:40:59,154 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:40:59,155 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T22:40:59,155 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-11T22:40:59,156 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-11T22:40:59,156 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:40:59,157 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T22:40:59,157 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-11T22:40:59,159 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/hbase/meta/1588230740 2024-12-11T22:40:59,161 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/hbase/meta/1588230740 2024-12-11T22:40:59,164 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-11T22:40:59,164 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-11T22:40:59,165 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-11T22:40:59,168 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-11T22:40:59,170 INFO [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72399152, jitterRate=0.07883143424987793}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-11T22:40:59,170 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-11T22:40:59,171 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733956859145Writing region info on filesystem at 1733956859145Initializing all the Stores at 1733956859147 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733956859147Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733956859147Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733956859148 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733956859148Cleaning up temporary data from old regions at 1733956859164 (+16 ms)Running coprocessor post-open hooks at 1733956859170 (+6 ms)Region opened successfully at 1733956859171 (+1 ms) 2024-12-11T22:40:59,177 INFO [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733956859083 2024-12-11T22:40:59,188 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-11T22:40:59,188 INFO [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-11T22:40:59,191 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=56d7a8405f4a,36971,1733956857186 2024-12-11T22:40:59,195 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 56d7a8405f4a,36971,1733956857186, state=OPEN 2024-12-11T22:40:59,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:40:59,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36971-0x1001717a55e0003, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:40:59,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:40:59,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:40:59,230 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:40:59,230 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:40:59,230 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:40:59,230 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:40:59,231 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=56d7a8405f4a,36971,1733956857186 2024-12-11T22:40:59,236 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-11T22:40:59,236 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,36971,1733956857186 in 319 msec 2024-12-11T22:40:59,243 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-11T22:40:59,243 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 864 msec 2024-12-11T22:40:59,245 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-11T22:40:59,245 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-11T22:40:59,263 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T22:40:59,265 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=56d7a8405f4a,36971,1733956857186, seqNum=-1] 2024-12-11T22:40:59,283 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T22:40:59,285 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58809, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T22:40:59,303 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1530 sec 2024-12-11T22:40:59,303 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733956859303, completionTime=-1 2024-12-11T22:40:59,306 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-11T22:40:59,306 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-11T22:40:59,351 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-11T22:40:59,351 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733956919351 2024-12-11T22:40:59,351 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733956979351 2024-12-11T22:40:59,351 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 44 msec 2024-12-11T22:40:59,353 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-11T22:40:59,361 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=56d7a8405f4a,34843,1733956856287-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T22:40:59,361 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=56d7a8405f4a,34843,1733956856287-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T22:40:59,361 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=56d7a8405f4a,34843,1733956856287-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T22:40:59,363 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-56d7a8405f4a:34843, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T22:40:59,364 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-11T22:40:59,364 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-11T22:40:59,371 DEBUG [master/56d7a8405f4a:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-11T22:40:59,400 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.118sec 2024-12-11T22:40:59,402 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-11T22:40:59,403 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-11T22:40:59,405 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-11T22:40:59,405 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-11T22:40:59,405 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-11T22:40:59,406 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=56d7a8405f4a,34843,1733956856287-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-11T22:40:59,407 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=56d7a8405f4a,34843,1733956856287-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-11T22:40:59,411 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-11T22:40:59,412 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-11T22:40:59,413 INFO [master/56d7a8405f4a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=56d7a8405f4a,34843,1733956856287-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T22:40:59,431 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ff0a408, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T22:40:59,436 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-11T22:40:59,436 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-11T22:40:59,440 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 56d7a8405f4a,34843,-1 for getting cluster id 2024-12-11T22:40:59,442 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T22:40:59,449 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '520b6ef7-849a-4c82-8e75-768f6964b41e' 2024-12-11T22:40:59,451 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T22:40:59,451 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "520b6ef7-849a-4c82-8e75-768f6964b41e" 2024-12-11T22:40:59,453 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c41e13e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T22:40:59,453 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [56d7a8405f4a,34843,-1] 2024-12-11T22:40:59,455 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T22:40:59,457 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T22:40:59,459 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T22:40:59,461 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6238c979, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T22:40:59,462 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T22:40:59,468 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=56d7a8405f4a,36971,1733956857186, seqNum=-1] 2024-12-11T22:40:59,468 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T22:40:59,470 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59184, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T22:40:59,488 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=56d7a8405f4a,34843,1733956856287 2024-12-11T22:40:59,491 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-11T22:40:59,495 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 56d7a8405f4a,34843,1733956856287 2024-12-11T22:40:59,497 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@27b83eb1 2024-12-11T22:40:59,498 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-11T22:40:59,500 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-11T22:40:59,506 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T22:40:59,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-12-11T22:40:59,516 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-12-11T22:40:59,519 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-12-11T22:40:59,519 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:40:59,522 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-11T22:40:59,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-11T22:40:59,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741837_1013 (size=358) 2024-12-11T22:40:59,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741837_1013 (size=358) 2024-12-11T22:40:59,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741837_1013 (size=358) 2024-12-11T22:40:59,541 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5a93ec512c97cf46e8942b91f4c8f088, NAME => 'test,,1733956859502.5a93ec512c97cf46e8942b91f4c8f088.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c 2024-12-11T22:40:59,541 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 6801ac4c1cb1416e045d462c592d2dc5, NAME => 'test,ccc,1733956859502.6801ac4c1cb1416e045d462c592d2dc5.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c 2024-12-11T22:40:59,542 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 66aac9e74e8263b0b8914fdc86eec66c, NAME => 'test,bbb,1733956859502.66aac9e74e8263b0b8914fdc86eec66c.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c 2024-12-11T22:40:59,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741840_1016 (size=42) 2024-12-11T22:40:59,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741838_1014 (size=45) 2024-12-11T22:40:59,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741840_1016 (size=42) 2024-12-11T22:40:59,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741840_1016 (size=42) 2024-12-11T22:40:59,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741838_1014 (size=45) 2024-12-11T22:40:59,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741838_1014 (size=45) 2024-12-11T22:40:59,564 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1733956859502.5a93ec512c97cf46e8942b91f4c8f088.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:40:59,564 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1733956859502.66aac9e74e8263b0b8914fdc86eec66c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:40:59,565 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 5a93ec512c97cf46e8942b91f4c8f088, disabling compactions & flushes 2024-12-11T22:40:59,565 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 66aac9e74e8263b0b8914fdc86eec66c, disabling compactions & flushes 2024-12-11T22:40:59,565 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1733956859502.5a93ec512c97cf46e8942b91f4c8f088. 2024-12-11T22:40:59,565 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1733956859502.66aac9e74e8263b0b8914fdc86eec66c. 2024-12-11T22:40:59,565 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1733956859502.5a93ec512c97cf46e8942b91f4c8f088. 2024-12-11T22:40:59,565 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1733956859502.66aac9e74e8263b0b8914fdc86eec66c. 2024-12-11T22:40:59,565 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1733956859502.5a93ec512c97cf46e8942b91f4c8f088. after waiting 0 ms 2024-12-11T22:40:59,565 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1733956859502.5a93ec512c97cf46e8942b91f4c8f088. 2024-12-11T22:40:59,565 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1733956859502.66aac9e74e8263b0b8914fdc86eec66c. after waiting 0 ms 2024-12-11T22:40:59,565 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1733956859502.5a93ec512c97cf46e8942b91f4c8f088. 2024-12-11T22:40:59,565 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1733956859502.66aac9e74e8263b0b8914fdc86eec66c. 2024-12-11T22:40:59,565 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5a93ec512c97cf46e8942b91f4c8f088: Waiting for close lock at 1733956859565Disabling compacts and flushes for region at 1733956859565Disabling writes for close at 1733956859565Writing region close event to WAL at 1733956859565Closed at 1733956859565 2024-12-11T22:40:59,565 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1733956859502.66aac9e74e8263b0b8914fdc86eec66c. 2024-12-11T22:40:59,565 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 66aac9e74e8263b0b8914fdc86eec66c: Waiting for close lock at 1733956859565Disabling compacts and flushes for region at 1733956859565Disabling writes for close at 1733956859565Writing region close event to WAL at 1733956859565Closed at 1733956859565 2024-12-11T22:40:59,565 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9f169feb3190772cec56c0ae51d57c26, NAME => 'test,ddd,1733956859502.9f169feb3190772cec56c0ae51d57c26.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c 2024-12-11T22:40:59,566 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => e9e94fba4d03035b5feaab607eb13853, NAME => 'test,eee,1733956859502.e9e94fba4d03035b5feaab607eb13853.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c 2024-12-11T22:40:59,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741839_1015 (size=45) 2024-12-11T22:40:59,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741839_1015 (size=45) 2024-12-11T22:40:59,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741839_1015 (size=45) 2024-12-11T22:40:59,571 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1733956859502.6801ac4c1cb1416e045d462c592d2dc5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:40:59,571 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 6801ac4c1cb1416e045d462c592d2dc5, disabling compactions & flushes 2024-12-11T22:40:59,571 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1733956859502.6801ac4c1cb1416e045d462c592d2dc5. 2024-12-11T22:40:59,571 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1733956859502.6801ac4c1cb1416e045d462c592d2dc5. 2024-12-11T22:40:59,571 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1733956859502.6801ac4c1cb1416e045d462c592d2dc5. after waiting 0 ms 2024-12-11T22:40:59,571 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1733956859502.6801ac4c1cb1416e045d462c592d2dc5. 2024-12-11T22:40:59,571 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1733956859502.6801ac4c1cb1416e045d462c592d2dc5. 2024-12-11T22:40:59,571 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 6801ac4c1cb1416e045d462c592d2dc5: Waiting for close lock at 1733956859571Disabling compacts and flushes for region at 1733956859571Disabling writes for close at 1733956859571Writing region close event to WAL at 1733956859571Closed at 1733956859571 2024-12-11T22:40:59,572 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 419f7368f61981f0a715c20304b74e0f, NAME => 'test,fff,1733956859502.419f7368f61981f0a715c20304b74e0f.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c 2024-12-11T22:40:59,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741843_1019 (size=45) 2024-12-11T22:40:59,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741843_1019 (size=45) 2024-12-11T22:40:59,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741843_1019 (size=45) 2024-12-11T22:40:59,587 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,fff,1733956859502.419f7368f61981f0a715c20304b74e0f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:40:59,587 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 419f7368f61981f0a715c20304b74e0f, disabling compactions & flushes 2024-12-11T22:40:59,587 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,fff,1733956859502.419f7368f61981f0a715c20304b74e0f. 2024-12-11T22:40:59,587 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1733956859502.419f7368f61981f0a715c20304b74e0f. 2024-12-11T22:40:59,587 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1733956859502.419f7368f61981f0a715c20304b74e0f. after waiting 0 ms 2024-12-11T22:40:59,587 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1733956859502.419f7368f61981f0a715c20304b74e0f. 2024-12-11T22:40:59,587 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,fff,1733956859502.419f7368f61981f0a715c20304b74e0f. 2024-12-11T22:40:59,587 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 419f7368f61981f0a715c20304b74e0f: Waiting for close lock at 1733956859587Disabling compacts and flushes for region at 1733956859587Disabling writes for close at 1733956859587Writing region close event to WAL at 1733956859587Closed at 1733956859587 2024-12-11T22:40:59,588 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => de3803c768bc063bdc71e495af58eb48, NAME => 'test,ggg,1733956859502.de3803c768bc063bdc71e495af58eb48.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c 2024-12-11T22:40:59,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741842_1018 (size=45) 2024-12-11T22:40:59,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741842_1018 (size=45) 2024-12-11T22:40:59,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741842_1018 (size=45) 2024-12-11T22:40:59,594 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ddd,1733956859502.9f169feb3190772cec56c0ae51d57c26.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:40:59,594 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 9f169feb3190772cec56c0ae51d57c26, disabling compactions & flushes 2024-12-11T22:40:59,594 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ddd,1733956859502.9f169feb3190772cec56c0ae51d57c26. 2024-12-11T22:40:59,594 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1733956859502.9f169feb3190772cec56c0ae51d57c26. 2024-12-11T22:40:59,594 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1733956859502.9f169feb3190772cec56c0ae51d57c26. after waiting 0 ms 2024-12-11T22:40:59,594 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1733956859502.9f169feb3190772cec56c0ae51d57c26. 2024-12-11T22:40:59,594 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ddd,1733956859502.9f169feb3190772cec56c0ae51d57c26. 2024-12-11T22:40:59,594 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9f169feb3190772cec56c0ae51d57c26: Waiting for close lock at 1733956859594Disabling compacts and flushes for region at 1733956859594Disabling writes for close at 1733956859594Writing region close event to WAL at 1733956859594Closed at 1733956859594 2024-12-11T22:40:59,595 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b31ee44b6609c355adc37986b13d239c, NAME => 'test,hhh,1733956859502.b31ee44b6609c355adc37986b13d239c.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c 2024-12-11T22:40:59,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741841_1017 (size=45) 2024-12-11T22:40:59,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741841_1017 (size=45) 2024-12-11T22:40:59,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741841_1017 (size=45) 2024-12-11T22:40:59,600 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,eee,1733956859502.e9e94fba4d03035b5feaab607eb13853.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:40:59,600 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing e9e94fba4d03035b5feaab607eb13853, disabling compactions & flushes 2024-12-11T22:40:59,600 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,eee,1733956859502.e9e94fba4d03035b5feaab607eb13853. 2024-12-11T22:40:59,600 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1733956859502.e9e94fba4d03035b5feaab607eb13853. 2024-12-11T22:40:59,600 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1733956859502.e9e94fba4d03035b5feaab607eb13853. after waiting 0 ms 2024-12-11T22:40:59,600 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1733956859502.e9e94fba4d03035b5feaab607eb13853. 2024-12-11T22:40:59,600 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,eee,1733956859502.e9e94fba4d03035b5feaab607eb13853. 2024-12-11T22:40:59,601 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for e9e94fba4d03035b5feaab607eb13853: Waiting for close lock at 1733956859600Disabling compacts and flushes for region at 1733956859600Disabling writes for close at 1733956859600Writing region close event to WAL at 1733956859600Closed at 1733956859600 2024-12-11T22:40:59,601 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 00fd1046c94e7f507c0ee43e6555d034, NAME => 'test,iii,1733956859502.00fd1046c94e7f507c0ee43e6555d034.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c 2024-12-11T22:40:59,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741844_1020 (size=45) 2024-12-11T22:40:59,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741844_1020 (size=45) 2024-12-11T22:40:59,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741844_1020 (size=45) 2024-12-11T22:40:59,608 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ggg,1733956859502.de3803c768bc063bdc71e495af58eb48.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:40:59,608 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing de3803c768bc063bdc71e495af58eb48, disabling compactions & flushes 2024-12-11T22:40:59,608 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ggg,1733956859502.de3803c768bc063bdc71e495af58eb48. 2024-12-11T22:40:59,608 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1733956859502.de3803c768bc063bdc71e495af58eb48. 2024-12-11T22:40:59,608 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1733956859502.de3803c768bc063bdc71e495af58eb48. after waiting 0 ms 2024-12-11T22:40:59,608 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1733956859502.de3803c768bc063bdc71e495af58eb48. 2024-12-11T22:40:59,609 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ggg,1733956859502.de3803c768bc063bdc71e495af58eb48. 2024-12-11T22:40:59,609 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for de3803c768bc063bdc71e495af58eb48: Waiting for close lock at 1733956859608Disabling compacts and flushes for region at 1733956859608Disabling writes for close at 1733956859608Writing region close event to WAL at 1733956859608Closed at 1733956859609 (+1 ms) 2024-12-11T22:40:59,609 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => ad4bb3db7f3f05f24b6115fc887d6ab5, NAME => 'test,jjj,1733956859502.ad4bb3db7f3f05f24b6115fc887d6ab5.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c 2024-12-11T22:40:59,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741845_1021 (size=45) 2024-12-11T22:40:59,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741845_1021 (size=45) 2024-12-11T22:40:59,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741845_1021 (size=45) 2024-12-11T22:40:59,620 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,hhh,1733956859502.b31ee44b6609c355adc37986b13d239c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:40:59,620 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing b31ee44b6609c355adc37986b13d239c, disabling compactions & flushes 2024-12-11T22:40:59,620 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,hhh,1733956859502.b31ee44b6609c355adc37986b13d239c. 2024-12-11T22:40:59,620 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1733956859502.b31ee44b6609c355adc37986b13d239c. 2024-12-11T22:40:59,620 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1733956859502.b31ee44b6609c355adc37986b13d239c. after waiting 0 ms 2024-12-11T22:40:59,620 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1733956859502.b31ee44b6609c355adc37986b13d239c. 2024-12-11T22:40:59,620 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,hhh,1733956859502.b31ee44b6609c355adc37986b13d239c. 2024-12-11T22:40:59,621 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for b31ee44b6609c355adc37986b13d239c: Waiting for close lock at 1733956859620Disabling compacts and flushes for region at 1733956859620Disabling writes for close at 1733956859620Writing region close event to WAL at 1733956859620Closed at 1733956859620 2024-12-11T22:40:59,621 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6561ead2eaa9f60da54d4ccff5474a28, NAME => 'test,kkk,1733956859502.6561ead2eaa9f60da54d4ccff5474a28.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c 2024-12-11T22:40:59,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741846_1022 (size=45) 2024-12-11T22:40:59,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741846_1022 (size=45) 2024-12-11T22:40:59,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741846_1022 (size=45) 2024-12-11T22:40:59,624 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,iii,1733956859502.00fd1046c94e7f507c0ee43e6555d034.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:40:59,624 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 00fd1046c94e7f507c0ee43e6555d034, disabling compactions & flushes 2024-12-11T22:40:59,624 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,iii,1733956859502.00fd1046c94e7f507c0ee43e6555d034. 2024-12-11T22:40:59,624 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1733956859502.00fd1046c94e7f507c0ee43e6555d034. 2024-12-11T22:40:59,624 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1733956859502.00fd1046c94e7f507c0ee43e6555d034. after waiting 0 ms 2024-12-11T22:40:59,624 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1733956859502.00fd1046c94e7f507c0ee43e6555d034. 2024-12-11T22:40:59,624 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,iii,1733956859502.00fd1046c94e7f507c0ee43e6555d034. 2024-12-11T22:40:59,624 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 00fd1046c94e7f507c0ee43e6555d034: Waiting for close lock at 1733956859624Disabling compacts and flushes for region at 1733956859624Disabling writes for close at 1733956859624Writing region close event to WAL at 1733956859624Closed at 1733956859624 2024-12-11T22:40:59,625 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 445df7fd625f94e0eb0712fcd777ebda, NAME => 'test,lll,1733956859502.445df7fd625f94e0eb0712fcd777ebda.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c 2024-12-11T22:40:59,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-11T22:40:59,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741847_1023 (size=45) 2024-12-11T22:40:59,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741847_1023 (size=45) 2024-12-11T22:40:59,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741847_1023 (size=45) 2024-12-11T22:40:59,640 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,jjj,1733956859502.ad4bb3db7f3f05f24b6115fc887d6ab5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:40:59,640 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing ad4bb3db7f3f05f24b6115fc887d6ab5, disabling compactions & flushes 2024-12-11T22:40:59,641 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,jjj,1733956859502.ad4bb3db7f3f05f24b6115fc887d6ab5. 2024-12-11T22:40:59,641 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1733956859502.ad4bb3db7f3f05f24b6115fc887d6ab5. 2024-12-11T22:40:59,641 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1733956859502.ad4bb3db7f3f05f24b6115fc887d6ab5. after waiting 0 ms 2024-12-11T22:40:59,641 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1733956859502.ad4bb3db7f3f05f24b6115fc887d6ab5. 2024-12-11T22:40:59,641 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,jjj,1733956859502.ad4bb3db7f3f05f24b6115fc887d6ab5. 2024-12-11T22:40:59,641 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for ad4bb3db7f3f05f24b6115fc887d6ab5: Waiting for close lock at 1733956859640Disabling compacts and flushes for region at 1733956859640Disabling writes for close at 1733956859641 (+1 ms)Writing region close event to WAL at 1733956859641Closed at 1733956859641 2024-12-11T22:40:59,642 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => f666d29f964372d8719d169fcecef641, NAME => 'test,mmm,1733956859502.f666d29f964372d8719d169fcecef641.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c 2024-12-11T22:40:59,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741849_1025 (size=45) 2024-12-11T22:40:59,647 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,lll,1733956859502.445df7fd625f94e0eb0712fcd777ebda.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:40:59,648 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 445df7fd625f94e0eb0712fcd777ebda, disabling compactions & flushes 2024-12-11T22:40:59,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741849_1025 (size=45) 2024-12-11T22:40:59,648 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,lll,1733956859502.445df7fd625f94e0eb0712fcd777ebda. 2024-12-11T22:40:59,648 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1733956859502.445df7fd625f94e0eb0712fcd777ebda. 2024-12-11T22:40:59,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741849_1025 (size=45) 2024-12-11T22:40:59,648 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1733956859502.445df7fd625f94e0eb0712fcd777ebda. after waiting 0 ms 2024-12-11T22:40:59,648 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1733956859502.445df7fd625f94e0eb0712fcd777ebda. 2024-12-11T22:40:59,648 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,lll,1733956859502.445df7fd625f94e0eb0712fcd777ebda. 2024-12-11T22:40:59,648 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 445df7fd625f94e0eb0712fcd777ebda: Waiting for close lock at 1733956859648Disabling compacts and flushes for region at 1733956859648Disabling writes for close at 1733956859648Writing region close event to WAL at 1733956859648Closed at 1733956859648 2024-12-11T22:40:59,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741848_1024 (size=45) 2024-12-11T22:40:59,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741848_1024 (size=45) 2024-12-11T22:40:59,650 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 65f30d8a32c00877e2cf6f22e7f50620, NAME => 'test,nnn,1733956859502.65f30d8a32c00877e2cf6f22e7f50620.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c 2024-12-11T22:40:59,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741848_1024 (size=45) 2024-12-11T22:40:59,651 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,kkk,1733956859502.6561ead2eaa9f60da54d4ccff5474a28.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:40:59,652 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 6561ead2eaa9f60da54d4ccff5474a28, disabling compactions & flushes 2024-12-11T22:40:59,652 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,kkk,1733956859502.6561ead2eaa9f60da54d4ccff5474a28. 2024-12-11T22:40:59,652 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1733956859502.6561ead2eaa9f60da54d4ccff5474a28. 2024-12-11T22:40:59,652 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1733956859502.6561ead2eaa9f60da54d4ccff5474a28. after waiting 0 ms 2024-12-11T22:40:59,652 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1733956859502.6561ead2eaa9f60da54d4ccff5474a28. 2024-12-11T22:40:59,652 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,kkk,1733956859502.6561ead2eaa9f60da54d4ccff5474a28. 2024-12-11T22:40:59,652 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6561ead2eaa9f60da54d4ccff5474a28: Waiting for close lock at 1733956859651Disabling compacts and flushes for region at 1733956859651Disabling writes for close at 1733956859652 (+1 ms)Writing region close event to WAL at 1733956859652Closed at 1733956859652 2024-12-11T22:40:59,653 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b1a38f11b48b53983535b863fd4392ff, NAME => 'test,ooo,1733956859502.b1a38f11b48b53983535b863fd4392ff.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c 2024-12-11T22:40:59,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741850_1026 (size=45) 2024-12-11T22:40:59,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741850_1026 (size=45) 2024-12-11T22:40:59,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741850_1026 (size=45) 2024-12-11T22:40:59,658 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,mmm,1733956859502.f666d29f964372d8719d169fcecef641.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:40:59,658 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing f666d29f964372d8719d169fcecef641, disabling compactions & flushes 2024-12-11T22:40:59,658 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,mmm,1733956859502.f666d29f964372d8719d169fcecef641. 2024-12-11T22:40:59,658 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1733956859502.f666d29f964372d8719d169fcecef641. 2024-12-11T22:40:59,658 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1733956859502.f666d29f964372d8719d169fcecef641. after waiting 0 ms 2024-12-11T22:40:59,658 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1733956859502.f666d29f964372d8719d169fcecef641. 2024-12-11T22:40:59,658 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,mmm,1733956859502.f666d29f964372d8719d169fcecef641. 2024-12-11T22:40:59,659 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for f666d29f964372d8719d169fcecef641: Waiting for close lock at 1733956859658Disabling compacts and flushes for region at 1733956859658Disabling writes for close at 1733956859658Writing region close event to WAL at 1733956859658Closed at 1733956859658 2024-12-11T22:40:59,659 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 01df3c1c862de5abd063d235e117a115, NAME => 'test,ppp,1733956859502.01df3c1c862de5abd063d235e117a115.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c 2024-12-11T22:40:59,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741851_1027 (size=45) 2024-12-11T22:40:59,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741851_1027 (size=45) 2024-12-11T22:40:59,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741851_1027 (size=45) 2024-12-11T22:40:59,667 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,nnn,1733956859502.65f30d8a32c00877e2cf6f22e7f50620.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:40:59,667 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 65f30d8a32c00877e2cf6f22e7f50620, disabling compactions & flushes 2024-12-11T22:40:59,667 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,nnn,1733956859502.65f30d8a32c00877e2cf6f22e7f50620. 2024-12-11T22:40:59,668 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1733956859502.65f30d8a32c00877e2cf6f22e7f50620. 2024-12-11T22:40:59,668 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1733956859502.65f30d8a32c00877e2cf6f22e7f50620. after waiting 0 ms 2024-12-11T22:40:59,668 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1733956859502.65f30d8a32c00877e2cf6f22e7f50620. 2024-12-11T22:40:59,668 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,nnn,1733956859502.65f30d8a32c00877e2cf6f22e7f50620. 2024-12-11T22:40:59,668 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 65f30d8a32c00877e2cf6f22e7f50620: Waiting for close lock at 1733956859667Disabling compacts and flushes for region at 1733956859667Disabling writes for close at 1733956859668 (+1 ms)Writing region close event to WAL at 1733956859668Closed at 1733956859668 2024-12-11T22:40:59,668 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 2f551d316d038ca1e1403bd05f890572, NAME => 'test,qqq,1733956859502.2f551d316d038ca1e1403bd05f890572.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c 2024-12-11T22:40:59,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741852_1028 (size=45) 2024-12-11T22:40:59,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741852_1028 (size=45) 2024-12-11T22:40:59,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741852_1028 (size=45) 2024-12-11T22:40:59,675 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ooo,1733956859502.b1a38f11b48b53983535b863fd4392ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:40:59,675 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing b1a38f11b48b53983535b863fd4392ff, disabling compactions & flushes 2024-12-11T22:40:59,675 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ooo,1733956859502.b1a38f11b48b53983535b863fd4392ff. 2024-12-11T22:40:59,675 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1733956859502.b1a38f11b48b53983535b863fd4392ff. 2024-12-11T22:40:59,675 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1733956859502.b1a38f11b48b53983535b863fd4392ff. after waiting 0 ms 2024-12-11T22:40:59,675 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1733956859502.b1a38f11b48b53983535b863fd4392ff. 2024-12-11T22:40:59,675 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ooo,1733956859502.b1a38f11b48b53983535b863fd4392ff. 2024-12-11T22:40:59,676 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for b1a38f11b48b53983535b863fd4392ff: Waiting for close lock at 1733956859675Disabling compacts and flushes for region at 1733956859675Disabling writes for close at 1733956859675Writing region close event to WAL at 1733956859675Closed at 1733956859675 2024-12-11T22:40:59,676 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f219e06826a4a3ede110adacdf28f6f4, NAME => 'test,rrr,1733956859502.f219e06826a4a3ede110adacdf28f6f4.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c 2024-12-11T22:40:59,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741853_1029 (size=45) 2024-12-11T22:40:59,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741853_1029 (size=45) 2024-12-11T22:40:59,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741853_1029 (size=45) 2024-12-11T22:40:59,684 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ppp,1733956859502.01df3c1c862de5abd063d235e117a115.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:40:59,684 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 01df3c1c862de5abd063d235e117a115, disabling compactions & flushes 2024-12-11T22:40:59,684 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ppp,1733956859502.01df3c1c862de5abd063d235e117a115. 2024-12-11T22:40:59,684 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1733956859502.01df3c1c862de5abd063d235e117a115. 2024-12-11T22:40:59,684 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1733956859502.01df3c1c862de5abd063d235e117a115. after waiting 0 ms 2024-12-11T22:40:59,684 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1733956859502.01df3c1c862de5abd063d235e117a115. 2024-12-11T22:40:59,684 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ppp,1733956859502.01df3c1c862de5abd063d235e117a115. 2024-12-11T22:40:59,684 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 01df3c1c862de5abd063d235e117a115: Waiting for close lock at 1733956859684Disabling compacts and flushes for region at 1733956859684Disabling writes for close at 1733956859684Writing region close event to WAL at 1733956859684Closed at 1733956859684 2024-12-11T22:40:59,685 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => daa100e1546a41747fac83159daee190, NAME => 'test,sss,1733956859502.daa100e1546a41747fac83159daee190.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c 2024-12-11T22:40:59,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741854_1030 (size=45) 2024-12-11T22:40:59,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741854_1030 (size=45) 2024-12-11T22:40:59,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741854_1030 (size=45) 2024-12-11T22:40:59,691 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,qqq,1733956859502.2f551d316d038ca1e1403bd05f890572.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:40:59,691 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 2f551d316d038ca1e1403bd05f890572, disabling compactions & flushes 2024-12-11T22:40:59,691 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,qqq,1733956859502.2f551d316d038ca1e1403bd05f890572. 2024-12-11T22:40:59,691 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1733956859502.2f551d316d038ca1e1403bd05f890572. 2024-12-11T22:40:59,691 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1733956859502.2f551d316d038ca1e1403bd05f890572. after waiting 0 ms 2024-12-11T22:40:59,691 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1733956859502.2f551d316d038ca1e1403bd05f890572. 2024-12-11T22:40:59,691 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,qqq,1733956859502.2f551d316d038ca1e1403bd05f890572. 2024-12-11T22:40:59,691 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 2f551d316d038ca1e1403bd05f890572: Waiting for close lock at 1733956859691Disabling compacts and flushes for region at 1733956859691Disabling writes for close at 1733956859691Writing region close event to WAL at 1733956859691Closed at 1733956859691 2024-12-11T22:40:59,692 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => da518288db554a985096494f24adc59f, NAME => 'test,ttt,1733956859502.da518288db554a985096494f24adc59f.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c 2024-12-11T22:40:59,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741856_1032 (size=45) 2024-12-11T22:40:59,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741856_1032 (size=45) 2024-12-11T22:40:59,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741856_1032 (size=45) 2024-12-11T22:40:59,698 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,sss,1733956859502.daa100e1546a41747fac83159daee190.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:40:59,698 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing daa100e1546a41747fac83159daee190, disabling compactions & flushes 2024-12-11T22:40:59,698 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,sss,1733956859502.daa100e1546a41747fac83159daee190. 2024-12-11T22:40:59,699 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1733956859502.daa100e1546a41747fac83159daee190. 2024-12-11T22:40:59,699 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1733956859502.daa100e1546a41747fac83159daee190. after waiting 0 ms 2024-12-11T22:40:59,699 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1733956859502.daa100e1546a41747fac83159daee190. 2024-12-11T22:40:59,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741855_1031 (size=45) 2024-12-11T22:40:59,699 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,sss,1733956859502.daa100e1546a41747fac83159daee190. 2024-12-11T22:40:59,699 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for daa100e1546a41747fac83159daee190: Waiting for close lock at 1733956859698Disabling compacts and flushes for region at 1733956859698Disabling writes for close at 1733956859699 (+1 ms)Writing region close event to WAL at 1733956859699Closed at 1733956859699 2024-12-11T22:40:59,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741855_1031 (size=45) 2024-12-11T22:40:59,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741855_1031 (size=45) 2024-12-11T22:40:59,700 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => f7dbe11f45ae33a883424947e613c7c6, NAME => 'test,uuu,1733956859502.f7dbe11f45ae33a883424947e613c7c6.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c 2024-12-11T22:40:59,700 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,rrr,1733956859502.f219e06826a4a3ede110adacdf28f6f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:40:59,700 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing f219e06826a4a3ede110adacdf28f6f4, disabling compactions & flushes 2024-12-11T22:40:59,700 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,rrr,1733956859502.f219e06826a4a3ede110adacdf28f6f4. 2024-12-11T22:40:59,700 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1733956859502.f219e06826a4a3ede110adacdf28f6f4. 2024-12-11T22:40:59,700 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1733956859502.f219e06826a4a3ede110adacdf28f6f4. after waiting 0 ms 2024-12-11T22:40:59,700 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1733956859502.f219e06826a4a3ede110adacdf28f6f4. 2024-12-11T22:40:59,700 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,rrr,1733956859502.f219e06826a4a3ede110adacdf28f6f4. 2024-12-11T22:40:59,700 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for f219e06826a4a3ede110adacdf28f6f4: Waiting for close lock at 1733956859700Disabling compacts and flushes for region at 1733956859700Disabling writes for close at 1733956859700Writing region close event to WAL at 1733956859700Closed at 1733956859700 2024-12-11T22:40:59,701 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b30c1b01a8241f23347f03f93df7f090, NAME => 'test,vvv,1733956859502.b30c1b01a8241f23347f03f93df7f090.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c 2024-12-11T22:40:59,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741857_1033 (size=45) 2024-12-11T22:40:59,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741857_1033 (size=45) 2024-12-11T22:40:59,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741857_1033 (size=45) 2024-12-11T22:40:59,707 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ttt,1733956859502.da518288db554a985096494f24adc59f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:40:59,707 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing da518288db554a985096494f24adc59f, disabling compactions & flushes 2024-12-11T22:40:59,707 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ttt,1733956859502.da518288db554a985096494f24adc59f. 2024-12-11T22:40:59,707 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1733956859502.da518288db554a985096494f24adc59f. 2024-12-11T22:40:59,707 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1733956859502.da518288db554a985096494f24adc59f. after waiting 0 ms 2024-12-11T22:40:59,707 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1733956859502.da518288db554a985096494f24adc59f. 2024-12-11T22:40:59,707 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ttt,1733956859502.da518288db554a985096494f24adc59f. 2024-12-11T22:40:59,707 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for da518288db554a985096494f24adc59f: Waiting for close lock at 1733956859707Disabling compacts and flushes for region at 1733956859707Disabling writes for close at 1733956859707Writing region close event to WAL at 1733956859707Closed at 1733956859707 2024-12-11T22:40:59,708 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 860d604009b38831efaa4f23b2d17596, NAME => 'test,www,1733956859502.860d604009b38831efaa4f23b2d17596.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c 2024-12-11T22:40:59,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741859_1035 (size=45) 2024-12-11T22:40:59,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741859_1035 (size=45) 2024-12-11T22:40:59,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741858_1034 (size=45) 2024-12-11T22:40:59,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741858_1034 (size=45) 2024-12-11T22:40:59,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741859_1035 (size=45) 2024-12-11T22:40:59,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741858_1034 (size=45) 2024-12-11T22:40:59,714 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,vvv,1733956859502.b30c1b01a8241f23347f03f93df7f090.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:40:59,714 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,uuu,1733956859502.f7dbe11f45ae33a883424947e613c7c6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:40:59,714 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing f7dbe11f45ae33a883424947e613c7c6, disabling compactions & flushes 2024-12-11T22:40:59,714 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing b30c1b01a8241f23347f03f93df7f090, disabling compactions & flushes 2024-12-11T22:40:59,714 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,uuu,1733956859502.f7dbe11f45ae33a883424947e613c7c6. 2024-12-11T22:40:59,714 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,vvv,1733956859502.b30c1b01a8241f23347f03f93df7f090. 2024-12-11T22:40:59,714 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1733956859502.f7dbe11f45ae33a883424947e613c7c6. 2024-12-11T22:40:59,714 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1733956859502.b30c1b01a8241f23347f03f93df7f090. 2024-12-11T22:40:59,715 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1733956859502.f7dbe11f45ae33a883424947e613c7c6. after waiting 0 ms 2024-12-11T22:40:59,715 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1733956859502.b30c1b01a8241f23347f03f93df7f090. after waiting 0 ms 2024-12-11T22:40:59,715 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1733956859502.f7dbe11f45ae33a883424947e613c7c6. 2024-12-11T22:40:59,715 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1733956859502.b30c1b01a8241f23347f03f93df7f090. 2024-12-11T22:40:59,715 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,vvv,1733956859502.b30c1b01a8241f23347f03f93df7f090. 2024-12-11T22:40:59,715 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,uuu,1733956859502.f7dbe11f45ae33a883424947e613c7c6. 2024-12-11T22:40:59,715 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for f7dbe11f45ae33a883424947e613c7c6: Waiting for close lock at 1733956859714Disabling compacts and flushes for region at 1733956859714Disabling writes for close at 1733956859715 (+1 ms)Writing region close event to WAL at 1733956859715Closed at 1733956859715 2024-12-11T22:40:59,715 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for b30c1b01a8241f23347f03f93df7f090: Waiting for close lock at 1733956859714Disabling compacts and flushes for region at 1733956859714Disabling writes for close at 1733956859715 (+1 ms)Writing region close event to WAL at 1733956859715Closed at 1733956859715 2024-12-11T22:40:59,715 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a0e02a7a4f8d57169d1f435eabc00839, NAME => 'test,xxx,1733956859502.a0e02a7a4f8d57169d1f435eabc00839.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c 2024-12-11T22:40:59,716 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => f19f540286846a30911c41a83b97457e, NAME => 'test,yyy,1733956859502.f19f540286846a30911c41a83b97457e.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c 2024-12-11T22:40:59,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741860_1036 (size=45) 2024-12-11T22:40:59,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741860_1036 (size=45) 2024-12-11T22:40:59,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741860_1036 (size=45) 2024-12-11T22:40:59,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741862_1038 (size=45) 2024-12-11T22:40:59,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741861_1037 (size=45) 2024-12-11T22:40:59,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741862_1038 (size=45) 2024-12-11T22:40:59,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741862_1038 (size=45) 2024-12-11T22:40:59,729 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,www,1733956859502.860d604009b38831efaa4f23b2d17596.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:40:59,730 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 860d604009b38831efaa4f23b2d17596, disabling compactions & flushes 2024-12-11T22:40:59,730 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,www,1733956859502.860d604009b38831efaa4f23b2d17596. 2024-12-11T22:40:59,730 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1733956859502.860d604009b38831efaa4f23b2d17596. 2024-12-11T22:40:59,730 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1733956859502.860d604009b38831efaa4f23b2d17596. after waiting 0 ms 2024-12-11T22:40:59,730 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1733956859502.860d604009b38831efaa4f23b2d17596. 2024-12-11T22:40:59,730 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,www,1733956859502.860d604009b38831efaa4f23b2d17596. 2024-12-11T22:40:59,730 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 860d604009b38831efaa4f23b2d17596: Waiting for close lock at 1733956859730Disabling compacts and flushes for region at 1733956859730Disabling writes for close at 1733956859730Writing region close event to WAL at 1733956859730Closed at 1733956859730 2024-12-11T22:40:59,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741861_1037 (size=45) 2024-12-11T22:40:59,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741861_1037 (size=45) 2024-12-11T22:40:59,730 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => ff9e1faa3192d62c84407efca19e0a89, NAME => 'test,zzz,1733956859502.ff9e1faa3192d62c84407efca19e0a89.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c 2024-12-11T22:40:59,731 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,yyy,1733956859502.f19f540286846a30911c41a83b97457e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:40:59,731 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,xxx,1733956859502.a0e02a7a4f8d57169d1f435eabc00839.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:40:59,731 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing f19f540286846a30911c41a83b97457e, disabling compactions & flushes 2024-12-11T22:40:59,731 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing a0e02a7a4f8d57169d1f435eabc00839, disabling compactions & flushes 2024-12-11T22:40:59,731 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,yyy,1733956859502.f19f540286846a30911c41a83b97457e. 2024-12-11T22:40:59,731 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1733956859502.f19f540286846a30911c41a83b97457e. 2024-12-11T22:40:59,731 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,xxx,1733956859502.a0e02a7a4f8d57169d1f435eabc00839. 2024-12-11T22:40:59,731 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1733956859502.f19f540286846a30911c41a83b97457e. after waiting 0 ms 2024-12-11T22:40:59,731 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1733956859502.a0e02a7a4f8d57169d1f435eabc00839. 2024-12-11T22:40:59,731 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1733956859502.f19f540286846a30911c41a83b97457e. 2024-12-11T22:40:59,731 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,yyy,1733956859502.f19f540286846a30911c41a83b97457e. 2024-12-11T22:40:59,731 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1733956859502.a0e02a7a4f8d57169d1f435eabc00839. after waiting 0 ms 2024-12-11T22:40:59,731 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for f19f540286846a30911c41a83b97457e: Waiting for close lock at 1733956859731Disabling compacts and flushes for region at 1733956859731Disabling writes for close at 1733956859731Writing region close event to WAL at 1733956859731Closed at 1733956859731 2024-12-11T22:40:59,731 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1733956859502.a0e02a7a4f8d57169d1f435eabc00839. 2024-12-11T22:40:59,731 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,xxx,1733956859502.a0e02a7a4f8d57169d1f435eabc00839. 2024-12-11T22:40:59,731 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for a0e02a7a4f8d57169d1f435eabc00839: Waiting for close lock at 1733956859731Disabling compacts and flushes for region at 1733956859731Disabling writes for close at 1733956859731Writing region close event to WAL at 1733956859731Closed at 1733956859731 2024-12-11T22:40:59,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741863_1039 (size=42) 2024-12-11T22:40:59,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741863_1039 (size=42) 2024-12-11T22:40:59,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741863_1039 (size=42) 2024-12-11T22:40:59,741 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,zzz,1733956859502.ff9e1faa3192d62c84407efca19e0a89.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:40:59,741 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing ff9e1faa3192d62c84407efca19e0a89, disabling compactions & flushes 2024-12-11T22:40:59,741 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,zzz,1733956859502.ff9e1faa3192d62c84407efca19e0a89. 2024-12-11T22:40:59,741 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1733956859502.ff9e1faa3192d62c84407efca19e0a89. 2024-12-11T22:40:59,741 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1733956859502.ff9e1faa3192d62c84407efca19e0a89. after waiting 0 ms 2024-12-11T22:40:59,742 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1733956859502.ff9e1faa3192d62c84407efca19e0a89. 2024-12-11T22:40:59,742 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,zzz,1733956859502.ff9e1faa3192d62c84407efca19e0a89. 2024-12-11T22:40:59,742 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for ff9e1faa3192d62c84407efca19e0a89: Waiting for close lock at 1733956859741Disabling compacts and flushes for region at 1733956859741Disabling writes for close at 1733956859742 (+1 ms)Writing region close event to WAL at 1733956859742Closed at 1733956859742 2024-12-11T22:40:59,744 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-12-11T22:40:59,750 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1733956859502.5a93ec512c97cf46e8942b91f4c8f088.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733956859744"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733956859744"}]},"ts":"1733956859744"} 2024-12-11T22:40:59,750 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1733956859502.66aac9e74e8263b0b8914fdc86eec66c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733956859744"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733956859744"}]},"ts":"1733956859744"} 2024-12-11T22:40:59,750 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1733956859502.6801ac4c1cb1416e045d462c592d2dc5.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733956859744"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733956859744"}]},"ts":"1733956859744"} 2024-12-11T22:40:59,750 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1733956859502.419f7368f61981f0a715c20304b74e0f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733956859744"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733956859744"}]},"ts":"1733956859744"} 2024-12-11T22:40:59,750 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1733956859502.9f169feb3190772cec56c0ae51d57c26.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733956859744"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733956859744"}]},"ts":"1733956859744"} 2024-12-11T22:40:59,750 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1733956859502.e9e94fba4d03035b5feaab607eb13853.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733956859744"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733956859744"}]},"ts":"1733956859744"} 2024-12-11T22:40:59,751 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1733956859502.de3803c768bc063bdc71e495af58eb48.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733956859744"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733956859744"}]},"ts":"1733956859744"} 2024-12-11T22:40:59,751 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1733956859502.b31ee44b6609c355adc37986b13d239c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733956859744"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733956859744"}]},"ts":"1733956859744"} 2024-12-11T22:40:59,751 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1733956859502.00fd1046c94e7f507c0ee43e6555d034.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733956859744"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733956859744"}]},"ts":"1733956859744"} 2024-12-11T22:40:59,751 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1733956859502.ad4bb3db7f3f05f24b6115fc887d6ab5.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733956859744"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733956859744"}]},"ts":"1733956859744"} 2024-12-11T22:40:59,751 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1733956859502.445df7fd625f94e0eb0712fcd777ebda.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733956859744"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733956859744"}]},"ts":"1733956859744"} 2024-12-11T22:40:59,751 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1733956859502.6561ead2eaa9f60da54d4ccff5474a28.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733956859744"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733956859744"}]},"ts":"1733956859744"} 2024-12-11T22:40:59,751 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1733956859502.f666d29f964372d8719d169fcecef641.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733956859744"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733956859744"}]},"ts":"1733956859744"} 2024-12-11T22:40:59,751 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1733956859502.65f30d8a32c00877e2cf6f22e7f50620.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733956859744"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733956859744"}]},"ts":"1733956859744"} 2024-12-11T22:40:59,752 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1733956859502.b1a38f11b48b53983535b863fd4392ff.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733956859744"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733956859744"}]},"ts":"1733956859744"} 2024-12-11T22:40:59,752 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1733956859502.01df3c1c862de5abd063d235e117a115.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733956859744"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733956859744"}]},"ts":"1733956859744"} 2024-12-11T22:40:59,752 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1733956859502.2f551d316d038ca1e1403bd05f890572.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733956859744"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733956859744"}]},"ts":"1733956859744"} 2024-12-11T22:40:59,752 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1733956859502.daa100e1546a41747fac83159daee190.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733956859744"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733956859744"}]},"ts":"1733956859744"} 2024-12-11T22:40:59,752 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1733956859502.f219e06826a4a3ede110adacdf28f6f4.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733956859744"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733956859744"}]},"ts":"1733956859744"} 2024-12-11T22:40:59,752 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1733956859502.da518288db554a985096494f24adc59f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733956859744"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733956859744"}]},"ts":"1733956859744"} 2024-12-11T22:40:59,752 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1733956859502.f7dbe11f45ae33a883424947e613c7c6.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733956859744"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733956859744"}]},"ts":"1733956859744"} 2024-12-11T22:40:59,752 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1733956859502.b30c1b01a8241f23347f03f93df7f090.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733956859744"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733956859744"}]},"ts":"1733956859744"} 2024-12-11T22:40:59,752 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1733956859502.860d604009b38831efaa4f23b2d17596.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733956859744"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733956859744"}]},"ts":"1733956859744"} 2024-12-11T22:40:59,753 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1733956859502.f19f540286846a30911c41a83b97457e.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733956859744"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733956859744"}]},"ts":"1733956859744"} 2024-12-11T22:40:59,753 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1733956859502.a0e02a7a4f8d57169d1f435eabc00839.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733956859744"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733956859744"}]},"ts":"1733956859744"} 2024-12-11T22:40:59,753 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1733956859502.ff9e1faa3192d62c84407efca19e0a89.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733956859744"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733956859744"}]},"ts":"1733956859744"} 2024-12-11T22:40:59,792 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-12-11T22:40:59,794 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-11T22:40:59,797 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733956859795"}]},"ts":"1733956859795"} 2024-12-11T22:40:59,801 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-12-11T22:40:59,802 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {56d7a8405f4a=0} racks are {/default-rack=0} 2024-12-11T22:40:59,817 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-11T22:40:59,817 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-11T22:40:59,817 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-11T22:40:59,817 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-11T22:40:59,817 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-11T22:40:59,817 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-11T22:40:59,817 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-11T22:40:59,817 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-11T22:40:59,818 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-11T22:40:59,818 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-11T22:40:59,820 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=66aac9e74e8263b0b8914fdc86eec66c, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6801ac4c1cb1416e045d462c592d2dc5, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9f169feb3190772cec56c0ae51d57c26, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e9e94fba4d03035b5feaab607eb13853, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b31ee44b6609c355adc37986b13d239c, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=00fd1046c94e7f507c0ee43e6555d034, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6561ead2eaa9f60da54d4ccff5474a28, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=445df7fd625f94e0eb0712fcd777ebda, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f666d29f964372d8719d169fcecef641, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=65f30d8a32c00877e2cf6f22e7f50620, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b1a38f11b48b53983535b863fd4392ff, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2f551d316d038ca1e1403bd05f890572, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=daa100e1546a41747fac83159daee190, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=da518288db554a985096494f24adc59f, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f7dbe11f45ae33a883424947e613c7c6, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=860d604009b38831efaa4f23b2d17596, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f19f540286846a30911c41a83b97457e, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ff9e1faa3192d62c84407efca19e0a89, ASSIGN}] 2024-12-11T22:40:59,830 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ff9e1faa3192d62c84407efca19e0a89, ASSIGN 2024-12-11T22:40:59,831 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ff9e1faa3192d62c84407efca19e0a89, ASSIGN; state=OFFLINE, location=56d7a8405f4a,40421,1733956857131; forceNewPlan=false, retain=false 2024-12-11T22:40:59,833 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f19f540286846a30911c41a83b97457e, ASSIGN 2024-12-11T22:40:59,834 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f19f540286846a30911c41a83b97457e, ASSIGN; state=OFFLINE, location=56d7a8405f4a,34703,1733956857011; forceNewPlan=false, retain=false 2024-12-11T22:40:59,835 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN 2024-12-11T22:40:59,837 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN; state=OFFLINE, location=56d7a8405f4a,36971,1733956857186; forceNewPlan=false, retain=false 2024-12-11T22:40:59,838 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=860d604009b38831efaa4f23b2d17596, ASSIGN 2024-12-11T22:40:59,839 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=860d604009b38831efaa4f23b2d17596, ASSIGN; state=OFFLINE, location=56d7a8405f4a,40421,1733956857131; forceNewPlan=false, retain=false 2024-12-11T22:40:59,840 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN 2024-12-11T22:40:59,841 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN; state=OFFLINE, location=56d7a8405f4a,36971,1733956857186; forceNewPlan=false, retain=false 2024-12-11T22:40:59,843 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f7dbe11f45ae33a883424947e613c7c6, ASSIGN 2024-12-11T22:40:59,844 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f7dbe11f45ae33a883424947e613c7c6, ASSIGN; state=OFFLINE, location=56d7a8405f4a,34703,1733956857011; forceNewPlan=false, retain=false 2024-12-11T22:40:59,845 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=da518288db554a985096494f24adc59f, ASSIGN 2024-12-11T22:40:59,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-11T22:40:59,846 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=da518288db554a985096494f24adc59f, ASSIGN; state=OFFLINE, location=56d7a8405f4a,40421,1733956857131; forceNewPlan=false, retain=false 2024-12-11T22:40:59,848 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=daa100e1546a41747fac83159daee190, ASSIGN 2024-12-11T22:40:59,849 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=daa100e1546a41747fac83159daee190, ASSIGN; state=OFFLINE, location=56d7a8405f4a,34703,1733956857011; forceNewPlan=false, retain=false 2024-12-11T22:40:59,851 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN 2024-12-11T22:40:59,852 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN; state=OFFLINE, location=56d7a8405f4a,36971,1733956857186; forceNewPlan=false, retain=false 2024-12-11T22:40:59,853 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2f551d316d038ca1e1403bd05f890572, ASSIGN 2024-12-11T22:40:59,855 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2f551d316d038ca1e1403bd05f890572, ASSIGN; state=OFFLINE, location=56d7a8405f4a,40421,1733956857131; forceNewPlan=false, retain=false 2024-12-11T22:40:59,856 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN 2024-12-11T22:40:59,857 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN; state=OFFLINE, location=56d7a8405f4a,36971,1733956857186; forceNewPlan=false, retain=false 2024-12-11T22:40:59,859 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b1a38f11b48b53983535b863fd4392ff, ASSIGN 2024-12-11T22:40:59,860 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b1a38f11b48b53983535b863fd4392ff, ASSIGN; state=OFFLINE, location=56d7a8405f4a,34703,1733956857011; forceNewPlan=false, retain=false 2024-12-11T22:40:59,862 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=65f30d8a32c00877e2cf6f22e7f50620, ASSIGN 2024-12-11T22:40:59,864 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=65f30d8a32c00877e2cf6f22e7f50620, ASSIGN; state=OFFLINE, location=56d7a8405f4a,40421,1733956857131; forceNewPlan=false, retain=false 2024-12-11T22:40:59,865 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f666d29f964372d8719d169fcecef641, ASSIGN 2024-12-11T22:40:59,866 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f666d29f964372d8719d169fcecef641, ASSIGN; state=OFFLINE, location=56d7a8405f4a,36971,1733956857186; forceNewPlan=false, retain=false 2024-12-11T22:40:59,868 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=445df7fd625f94e0eb0712fcd777ebda, ASSIGN 2024-12-11T22:40:59,869 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=445df7fd625f94e0eb0712fcd777ebda, ASSIGN; state=OFFLINE, location=56d7a8405f4a,40421,1733956857131; forceNewPlan=false, retain=false 2024-12-11T22:40:59,871 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6561ead2eaa9f60da54d4ccff5474a28, ASSIGN 2024-12-11T22:40:59,872 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6561ead2eaa9f60da54d4ccff5474a28, ASSIGN; state=OFFLINE, location=56d7a8405f4a,34703,1733956857011; forceNewPlan=false, retain=false 2024-12-11T22:40:59,873 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN 2024-12-11T22:40:59,875 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN; state=OFFLINE, location=56d7a8405f4a,36971,1733956857186; forceNewPlan=false, retain=false 2024-12-11T22:40:59,876 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=00fd1046c94e7f507c0ee43e6555d034, ASSIGN 2024-12-11T22:40:59,877 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=00fd1046c94e7f507c0ee43e6555d034, ASSIGN; state=OFFLINE, location=56d7a8405f4a,34703,1733956857011; forceNewPlan=false, retain=false 2024-12-11T22:40:59,878 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b31ee44b6609c355adc37986b13d239c, ASSIGN 2024-12-11T22:40:59,879 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b31ee44b6609c355adc37986b13d239c, ASSIGN; state=OFFLINE, location=56d7a8405f4a,40421,1733956857131; forceNewPlan=false, retain=false 2024-12-11T22:40:59,880 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN 2024-12-11T22:40:59,881 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN; state=OFFLINE, location=56d7a8405f4a,36971,1733956857186; forceNewPlan=false, retain=false 2024-12-11T22:40:59,883 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN 2024-12-11T22:40:59,884 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN; state=OFFLINE, location=56d7a8405f4a,36971,1733956857186; forceNewPlan=false, retain=false 2024-12-11T22:40:59,885 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e9e94fba4d03035b5feaab607eb13853, ASSIGN 2024-12-11T22:40:59,887 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e9e94fba4d03035b5feaab607eb13853, ASSIGN; state=OFFLINE, location=56d7a8405f4a,34703,1733956857011; forceNewPlan=false, retain=false 2024-12-11T22:40:59,888 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9f169feb3190772cec56c0ae51d57c26, ASSIGN 2024-12-11T22:40:59,889 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9f169feb3190772cec56c0ae51d57c26, ASSIGN; state=OFFLINE, location=56d7a8405f4a,40421,1733956857131; forceNewPlan=false, retain=false 2024-12-11T22:40:59,891 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6801ac4c1cb1416e045d462c592d2dc5, ASSIGN 2024-12-11T22:40:59,892 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6801ac4c1cb1416e045d462c592d2dc5, ASSIGN; state=OFFLINE, location=56d7a8405f4a,34703,1733956857011; forceNewPlan=false, retain=false 2024-12-11T22:40:59,893 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=66aac9e74e8263b0b8914fdc86eec66c, ASSIGN 2024-12-11T22:40:59,894 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=66aac9e74e8263b0b8914fdc86eec66c, ASSIGN; state=OFFLINE, location=56d7a8405f4a,40421,1733956857131; forceNewPlan=false, retain=false 2024-12-11T22:40:59,895 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN 2024-12-11T22:40:59,896 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN; state=OFFLINE, location=56d7a8405f4a,36971,1733956857186; forceNewPlan=false, retain=false 2024-12-11T22:40:59,988 INFO [56d7a8405f4a:34843 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-12-11T22:40:59,989 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=ff9e1faa3192d62c84407efca19e0a89, regionState=OPENING, regionLocation=56d7a8405f4a,40421,1733956857131 2024-12-11T22:40:59,990 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=860d604009b38831efaa4f23b2d17596, regionState=OPENING, regionLocation=56d7a8405f4a,40421,1733956857131 2024-12-11T22:40:59,991 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=da518288db554a985096494f24adc59f, regionState=OPENING, regionLocation=56d7a8405f4a,40421,1733956857131 2024-12-11T22:40:59,991 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=2f551d316d038ca1e1403bd05f890572, regionState=OPENING, regionLocation=56d7a8405f4a,40421,1733956857131 2024-12-11T22:40:59,991 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=65f30d8a32c00877e2cf6f22e7f50620, regionState=OPENING, regionLocation=56d7a8405f4a,40421,1733956857131 2024-12-11T22:40:59,992 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=445df7fd625f94e0eb0712fcd777ebda, regionState=OPENING, regionLocation=56d7a8405f4a,40421,1733956857131 2024-12-11T22:40:59,992 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=b31ee44b6609c355adc37986b13d239c, regionState=OPENING, regionLocation=56d7a8405f4a,40421,1733956857131 2024-12-11T22:40:59,993 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=9f169feb3190772cec56c0ae51d57c26, regionState=OPENING, regionLocation=56d7a8405f4a,40421,1733956857131 2024-12-11T22:40:59,993 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ff9e1faa3192d62c84407efca19e0a89, ASSIGN because future has completed 2024-12-11T22:40:59,993 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=66aac9e74e8263b0b8914fdc86eec66c, regionState=OPENING, regionLocation=56d7a8405f4a,40421,1733956857131 2024-12-11T22:40:59,994 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=a0e02a7a4f8d57169d1f435eabc00839, regionState=OPENING, regionLocation=56d7a8405f4a,36971,1733956857186 2024-12-11T22:40:59,994 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff9e1faa3192d62c84407efca19e0a89, server=56d7a8405f4a,40421,1733956857131}] 2024-12-11T22:40:59,995 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=860d604009b38831efaa4f23b2d17596, ASSIGN because future has completed 2024-12-11T22:40:59,997 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=da518288db554a985096494f24adc59f, ASSIGN because future has completed 2024-12-11T22:40:59,998 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2f551d316d038ca1e1403bd05f890572, ASSIGN because future has completed 2024-12-11T22:41:00,000 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=65f30d8a32c00877e2cf6f22e7f50620, ASSIGN because future has completed 2024-12-11T22:41:00,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2f551d316d038ca1e1403bd05f890572, server=56d7a8405f4a,40421,1733956857131}] 2024-12-11T22:41:00,002 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=445df7fd625f94e0eb0712fcd777ebda, ASSIGN because future has completed 2024-12-11T22:41:00,004 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b31ee44b6609c355adc37986b13d239c, ASSIGN because future has completed 2024-12-11T22:41:00,006 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9f169feb3190772cec56c0ae51d57c26, ASSIGN because future has completed 2024-12-11T22:41:00,008 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9f169feb3190772cec56c0ae51d57c26, server=56d7a8405f4a,40421,1733956857131}] 2024-12-11T22:41:00,008 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=66aac9e74e8263b0b8914fdc86eec66c, ASSIGN because future has completed 2024-12-11T22:41:00,010 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN because future has completed 2024-12-11T22:41:00,014 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0e02a7a4f8d57169d1f435eabc00839, server=56d7a8405f4a,36971,1733956857186}] 2024-12-11T22:41:00,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66aac9e74e8263b0b8914fdc86eec66c, server=56d7a8405f4a,40421,1733956857131}] 2024-12-11T22:41:00,023 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure b31ee44b6609c355adc37986b13d239c, server=56d7a8405f4a,40421,1733956857131}] 2024-12-11T22:41:00,027 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 445df7fd625f94e0eb0712fcd777ebda, server=56d7a8405f4a,40421,1733956857131}] 2024-12-11T22:41:00,032 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 65f30d8a32c00877e2cf6f22e7f50620, server=56d7a8405f4a,40421,1733956857131}] 2024-12-11T22:41:00,035 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure da518288db554a985096494f24adc59f, server=56d7a8405f4a,40421,1733956857131}] 2024-12-11T22:41:00,039 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 860d604009b38831efaa4f23b2d17596, server=56d7a8405f4a,40421,1733956857131}] 2024-12-11T22:41:00,044 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=b30c1b01a8241f23347f03f93df7f090, regionState=OPENING, regionLocation=56d7a8405f4a,36971,1733956857186 2024-12-11T22:41:00,044 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=f219e06826a4a3ede110adacdf28f6f4, regionState=OPENING, regionLocation=56d7a8405f4a,36971,1733956857186 2024-12-11T22:41:00,045 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=01df3c1c862de5abd063d235e117a115, regionState=OPENING, regionLocation=56d7a8405f4a,36971,1733956857186 2024-12-11T22:41:00,046 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=f666d29f964372d8719d169fcecef641, regionState=OPENING, regionLocation=56d7a8405f4a,36971,1733956857186 2024-12-11T22:41:00,046 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=ad4bb3db7f3f05f24b6115fc887d6ab5, regionState=OPENING, regionLocation=56d7a8405f4a,36971,1733956857186 2024-12-11T22:41:00,047 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN because future has completed 2024-12-11T22:41:00,047 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=de3803c768bc063bdc71e495af58eb48, regionState=OPENING, regionLocation=56d7a8405f4a,36971,1733956857186 2024-12-11T22:41:00,047 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure b30c1b01a8241f23347f03f93df7f090, server=56d7a8405f4a,36971,1733956857186}] 2024-12-11T22:41:00,048 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN because future has completed 2024-12-11T22:41:00,050 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN because future has completed 2024-12-11T22:41:00,052 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f666d29f964372d8719d169fcecef641, ASSIGN because future has completed 2024-12-11T22:41:00,053 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure f666d29f964372d8719d169fcecef641, server=56d7a8405f4a,36971,1733956857186}] 2024-12-11T22:41:00,053 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN because future has completed 2024-12-11T22:41:00,055 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN because future has completed 2024-12-11T22:41:00,058 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure de3803c768bc063bdc71e495af58eb48, server=56d7a8405f4a,36971,1733956857186}] 2024-12-11T22:41:00,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure ad4bb3db7f3f05f24b6115fc887d6ab5, server=56d7a8405f4a,36971,1733956857186}] 2024-12-11T22:41:00,066 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 01df3c1c862de5abd063d235e117a115, server=56d7a8405f4a,36971,1733956857186}] 2024-12-11T22:41:00,071 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure f219e06826a4a3ede110adacdf28f6f4, server=56d7a8405f4a,36971,1733956857186}] 2024-12-11T22:41:00,075 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=419f7368f61981f0a715c20304b74e0f, regionState=OPENING, regionLocation=56d7a8405f4a,36971,1733956857186 2024-12-11T22:41:00,076 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5a93ec512c97cf46e8942b91f4c8f088, regionState=OPENING, regionLocation=56d7a8405f4a,36971,1733956857186 2024-12-11T22:41:00,076 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=f19f540286846a30911c41a83b97457e, regionState=OPENING, regionLocation=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:00,077 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=f7dbe11f45ae33a883424947e613c7c6, regionState=OPENING, regionLocation=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:00,077 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=daa100e1546a41747fac83159daee190, regionState=OPENING, regionLocation=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:00,078 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=b1a38f11b48b53983535b863fd4392ff, regionState=OPENING, regionLocation=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:00,078 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN because future has completed 2024-12-11T22:41:00,079 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 419f7368f61981f0a715c20304b74e0f, server=56d7a8405f4a,36971,1733956857186}] 2024-12-11T22:41:00,080 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN because future has completed 2024-12-11T22:41:00,082 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f19f540286846a30911c41a83b97457e, ASSIGN because future has completed 2024-12-11T22:41:00,084 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f7dbe11f45ae33a883424947e613c7c6, ASSIGN because future has completed 2024-12-11T22:41:00,085 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure f7dbe11f45ae33a883424947e613c7c6, server=56d7a8405f4a,34703,1733956857011}] 2024-12-11T22:41:00,085 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=daa100e1546a41747fac83159daee190, ASSIGN because future has completed 2024-12-11T22:41:00,086 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b1a38f11b48b53983535b863fd4392ff, ASSIGN because future has completed 2024-12-11T22:41:00,090 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure b1a38f11b48b53983535b863fd4392ff, server=56d7a8405f4a,34703,1733956857011}] 2024-12-11T22:41:00,094 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure daa100e1546a41747fac83159daee190, server=56d7a8405f4a,34703,1733956857011}] 2024-12-11T22:41:00,098 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure f19f540286846a30911c41a83b97457e, server=56d7a8405f4a,34703,1733956857011}] 2024-12-11T22:41:00,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,36971,1733956857186}] 2024-12-11T22:41:00,106 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=6561ead2eaa9f60da54d4ccff5474a28, regionState=OPENING, regionLocation=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:00,107 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=00fd1046c94e7f507c0ee43e6555d034, regionState=OPENING, regionLocation=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:00,107 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=e9e94fba4d03035b5feaab607eb13853, regionState=OPENING, regionLocation=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:00,108 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=6801ac4c1cb1416e045d462c592d2dc5, regionState=OPENING, regionLocation=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:00,109 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6561ead2eaa9f60da54d4ccff5474a28, ASSIGN because future has completed 2024-12-11T22:41:00,109 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6561ead2eaa9f60da54d4ccff5474a28, server=56d7a8405f4a,34703,1733956857011}] 2024-12-11T22:41:00,110 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=00fd1046c94e7f507c0ee43e6555d034, ASSIGN because future has completed 2024-12-11T22:41:00,112 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e9e94fba4d03035b5feaab607eb13853, ASSIGN because future has completed 2024-12-11T22:41:00,116 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6801ac4c1cb1416e045d462c592d2dc5, ASSIGN because future has completed 2024-12-11T22:41:00,117 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6801ac4c1cb1416e045d462c592d2dc5, server=56d7a8405f4a,34703,1733956857011}] 2024-12-11T22:41:00,121 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure e9e94fba4d03035b5feaab607eb13853, server=56d7a8405f4a,34703,1733956857011}] 2024-12-11T22:41:00,125 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 00fd1046c94e7f507c0ee43e6555d034, server=56d7a8405f4a,34703,1733956857011}] 2024-12-11T22:41:00,149 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-11T22:41:00,151 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53753, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-11T22:41:00,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-11T22:41:00,157 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,zzz,1733956859502.ff9e1faa3192d62c84407efca19e0a89. 2024-12-11T22:41:00,158 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => ff9e1faa3192d62c84407efca19e0a89, NAME => 'test,zzz,1733956859502.ff9e1faa3192d62c84407efca19e0a89.', STARTKEY => 'zzz', ENDKEY => ''} 2024-12-11T22:41:00,158 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ff9e1faa3192d62c84407efca19e0a89 2024-12-11T22:41:00,158 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,zzz,1733956859502.ff9e1faa3192d62c84407efca19e0a89.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:41:00,158 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for ff9e1faa3192d62c84407efca19e0a89 2024-12-11T22:41:00,159 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for ff9e1faa3192d62c84407efca19e0a89 2024-12-11T22:41:00,161 INFO [StoreOpener-ff9e1faa3192d62c84407efca19e0a89-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ff9e1faa3192d62c84407efca19e0a89 2024-12-11T22:41:00,163 INFO [StoreOpener-ff9e1faa3192d62c84407efca19e0a89-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff9e1faa3192d62c84407efca19e0a89 columnFamilyName family 2024-12-11T22:41:00,163 DEBUG [StoreOpener-ff9e1faa3192d62c84407efca19e0a89-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:41:00,164 INFO [StoreOpener-ff9e1faa3192d62c84407efca19e0a89-1 {}] regionserver.HStore(327): Store=ff9e1faa3192d62c84407efca19e0a89/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:41:00,165 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for ff9e1faa3192d62c84407efca19e0a89 2024-12-11T22:41:00,166 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/ff9e1faa3192d62c84407efca19e0a89 2024-12-11T22:41:00,166 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/ff9e1faa3192d62c84407efca19e0a89 2024-12-11T22:41:00,167 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for ff9e1faa3192d62c84407efca19e0a89 2024-12-11T22:41:00,167 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for ff9e1faa3192d62c84407efca19e0a89 2024-12-11T22:41:00,169 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for ff9e1faa3192d62c84407efca19e0a89 2024-12-11T22:41:00,172 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,vvv,1733956859502.b30c1b01a8241f23347f03f93df7f090. 2024-12-11T22:41:00,172 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => b30c1b01a8241f23347f03f93df7f090, NAME => 'test,vvv,1733956859502.b30c1b01a8241f23347f03f93df7f090.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-12-11T22:41:00,172 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/ff9e1faa3192d62c84407efca19e0a89/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T22:41:00,173 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b30c1b01a8241f23347f03f93df7f090 2024-12-11T22:41:00,173 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,vvv,1733956859502.b30c1b01a8241f23347f03f93df7f090.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:41:00,173 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for b30c1b01a8241f23347f03f93df7f090 2024-12-11T22:41:00,173 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for b30c1b01a8241f23347f03f93df7f090 2024-12-11T22:41:00,173 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened ff9e1faa3192d62c84407efca19e0a89; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71425480, jitterRate=0.0643225908279419}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:41:00,173 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ff9e1faa3192d62c84407efca19e0a89 2024-12-11T22:41:00,174 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for ff9e1faa3192d62c84407efca19e0a89: Running coprocessor pre-open hook at 1733956860159Writing region info on filesystem at 1733956860159Initializing all the Stores at 1733956860161 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733956860161Cleaning up temporary data from old regions at 1733956860167 (+6 ms)Running coprocessor post-open hooks at 1733956860173 (+6 ms)Region opened successfully at 1733956860174 (+1 ms) 2024-12-11T22:41:00,175 INFO [StoreOpener-b30c1b01a8241f23347f03f93df7f090-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b30c1b01a8241f23347f03f93df7f090 2024-12-11T22:41:00,176 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1733956859502.ff9e1faa3192d62c84407efca19e0a89., pid=31, masterSystemTime=1733956860148 2024-12-11T22:41:00,177 INFO [StoreOpener-b30c1b01a8241f23347f03f93df7f090-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b30c1b01a8241f23347f03f93df7f090 columnFamilyName family 2024-12-11T22:41:00,177 DEBUG [StoreOpener-b30c1b01a8241f23347f03f93df7f090-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:41:00,178 INFO [StoreOpener-b30c1b01a8241f23347f03f93df7f090-1 {}] regionserver.HStore(327): Store=b30c1b01a8241f23347f03f93df7f090/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:41:00,179 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for b30c1b01a8241f23347f03f93df7f090 2024-12-11T22:41:00,180 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1733956859502.ff9e1faa3192d62c84407efca19e0a89. 2024-12-11T22:41:00,180 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,zzz,1733956859502.ff9e1faa3192d62c84407efca19e0a89. 2024-12-11T22:41:00,180 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,ddd,1733956859502.9f169feb3190772cec56c0ae51d57c26. 2024-12-11T22:41:00,180 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 9f169feb3190772cec56c0ae51d57c26, NAME => 'test,ddd,1733956859502.9f169feb3190772cec56c0ae51d57c26.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-12-11T22:41:00,181 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/b30c1b01a8241f23347f03f93df7f090 2024-12-11T22:41:00,181 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=ff9e1faa3192d62c84407efca19e0a89, regionState=OPEN, openSeqNum=2, regionLocation=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:00,181 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 9f169feb3190772cec56c0ae51d57c26 2024-12-11T22:41:00,181 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,ddd,1733956859502.9f169feb3190772cec56c0ae51d57c26.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:41:00,181 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 9f169feb3190772cec56c0ae51d57c26 2024-12-11T22:41:00,181 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 9f169feb3190772cec56c0ae51d57c26 2024-12-11T22:41:00,181 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/b30c1b01a8241f23347f03f93df7f090 2024-12-11T22:41:00,183 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for b30c1b01a8241f23347f03f93df7f090 2024-12-11T22:41:00,183 INFO [StoreOpener-9f169feb3190772cec56c0ae51d57c26-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 9f169feb3190772cec56c0ae51d57c26 2024-12-11T22:41:00,183 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for b30c1b01a8241f23347f03f93df7f090 2024-12-11T22:41:00,184 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff9e1faa3192d62c84407efca19e0a89, server=56d7a8405f4a,40421,1733956857131 because future has completed 2024-12-11T22:41:00,185 INFO [StoreOpener-9f169feb3190772cec56c0ae51d57c26-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9f169feb3190772cec56c0ae51d57c26 columnFamilyName family 2024-12-11T22:41:00,185 DEBUG [StoreOpener-9f169feb3190772cec56c0ae51d57c26-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:41:00,186 INFO [StoreOpener-9f169feb3190772cec56c0ae51d57c26-1 {}] regionserver.HStore(327): Store=9f169feb3190772cec56c0ae51d57c26/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:41:00,186 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 9f169feb3190772cec56c0ae51d57c26 2024-12-11T22:41:00,187 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for b30c1b01a8241f23347f03f93df7f090 2024-12-11T22:41:00,188 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/9f169feb3190772cec56c0ae51d57c26 2024-12-11T22:41:00,189 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/9f169feb3190772cec56c0ae51d57c26 2024-12-11T22:41:00,190 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 9f169feb3190772cec56c0ae51d57c26 2024-12-11T22:41:00,190 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 9f169feb3190772cec56c0ae51d57c26 2024-12-11T22:41:00,191 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-12-11T22:41:00,191 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure ff9e1faa3192d62c84407efca19e0a89, server=56d7a8405f4a,40421,1733956857131 in 192 msec 2024-12-11T22:41:00,191 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/b30c1b01a8241f23347f03f93df7f090/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T22:41:00,192 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened b30c1b01a8241f23347f03f93df7f090; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67432998, jitterRate=0.004829972982406616}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:41:00,192 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b30c1b01a8241f23347f03f93df7f090 2024-12-11T22:41:00,192 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for b30c1b01a8241f23347f03f93df7f090: Running coprocessor pre-open hook at 1733956860173Writing region info on filesystem at 1733956860173Initializing all the Stores at 1733956860175 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733956860175Cleaning up temporary data from old regions at 1733956860183 (+8 ms)Running coprocessor post-open hooks at 1733956860192 (+9 ms)Region opened successfully at 1733956860192 2024-12-11T22:41:00,194 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 9f169feb3190772cec56c0ae51d57c26 2024-12-11T22:41:00,194 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1733956859502.b30c1b01a8241f23347f03f93df7f090., pid=41, masterSystemTime=1733956860167 2024-12-11T22:41:00,194 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ff9e1faa3192d62c84407efca19e0a89, ASSIGN in 372 msec 2024-12-11T22:41:00,197 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1733956859502.b30c1b01a8241f23347f03f93df7f090. 2024-12-11T22:41:00,197 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,vvv,1733956859502.b30c1b01a8241f23347f03f93df7f090. 2024-12-11T22:41:00,197 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,ggg,1733956859502.de3803c768bc063bdc71e495af58eb48. 2024-12-11T22:41:00,197 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => de3803c768bc063bdc71e495af58eb48, NAME => 'test,ggg,1733956859502.de3803c768bc063bdc71e495af58eb48.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-12-11T22:41:00,197 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=b30c1b01a8241f23347f03f93df7f090, regionState=OPEN, openSeqNum=2, regionLocation=56d7a8405f4a,36971,1733956857186 2024-12-11T22:41:00,197 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test de3803c768bc063bdc71e495af58eb48 2024-12-11T22:41:00,198 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,ggg,1733956859502.de3803c768bc063bdc71e495af58eb48.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:41:00,198 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for de3803c768bc063bdc71e495af58eb48 2024-12-11T22:41:00,198 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for de3803c768bc063bdc71e495af58eb48 2024-12-11T22:41:00,198 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/9f169feb3190772cec56c0ae51d57c26/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T22:41:00,200 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 9f169feb3190772cec56c0ae51d57c26; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68729866, jitterRate=0.02415481209754944}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:41:00,200 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9f169feb3190772cec56c0ae51d57c26 2024-12-11T22:41:00,200 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 9f169feb3190772cec56c0ae51d57c26: Running coprocessor pre-open hook at 1733956860181Writing region info on filesystem at 1733956860181Initializing all the Stores at 1733956860183 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733956860183Cleaning up temporary data from old regions at 1733956860190 (+7 ms)Running coprocessor post-open hooks at 1733956860200 (+10 ms)Region opened successfully at 1733956860200 2024-12-11T22:41:00,200 INFO [StoreOpener-de3803c768bc063bdc71e495af58eb48-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region de3803c768bc063bdc71e495af58eb48 2024-12-11T22:41:00,201 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure b30c1b01a8241f23347f03f93df7f090, server=56d7a8405f4a,36971,1733956857186 because future has completed 2024-12-11T22:41:00,201 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1733956859502.9f169feb3190772cec56c0ae51d57c26., pid=33, masterSystemTime=1733956860148 2024-12-11T22:41:00,204 INFO [StoreOpener-de3803c768bc063bdc71e495af58eb48-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de3803c768bc063bdc71e495af58eb48 columnFamilyName family 2024-12-11T22:41:00,204 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1733956859502.9f169feb3190772cec56c0ae51d57c26. 2024-12-11T22:41:00,204 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,ddd,1733956859502.9f169feb3190772cec56c0ae51d57c26. 2024-12-11T22:41:00,204 DEBUG [StoreOpener-de3803c768bc063bdc71e495af58eb48-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:41:00,205 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,ttt,1733956859502.da518288db554a985096494f24adc59f. 2024-12-11T22:41:00,205 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => da518288db554a985096494f24adc59f, NAME => 'test,ttt,1733956859502.da518288db554a985096494f24adc59f.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-12-11T22:41:00,205 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test da518288db554a985096494f24adc59f 2024-12-11T22:41:00,205 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,ttt,1733956859502.da518288db554a985096494f24adc59f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:41:00,205 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for da518288db554a985096494f24adc59f 2024-12-11T22:41:00,205 INFO [StoreOpener-de3803c768bc063bdc71e495af58eb48-1 {}] regionserver.HStore(327): Store=de3803c768bc063bdc71e495af58eb48/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:41:00,206 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for da518288db554a985096494f24adc59f 2024-12-11T22:41:00,206 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for de3803c768bc063bdc71e495af58eb48 2024-12-11T22:41:00,206 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=26 2024-12-11T22:41:00,207 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure b30c1b01a8241f23347f03f93df7f090, server=56d7a8405f4a,36971,1733956857186 in 155 msec 2024-12-11T22:41:00,207 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/de3803c768bc063bdc71e495af58eb48 2024-12-11T22:41:00,207 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/de3803c768bc063bdc71e495af58eb48 2024-12-11T22:41:00,208 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for de3803c768bc063bdc71e495af58eb48 2024-12-11T22:41:00,208 INFO [StoreOpener-da518288db554a985096494f24adc59f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region da518288db554a985096494f24adc59f 2024-12-11T22:41:00,209 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for de3803c768bc063bdc71e495af58eb48 2024-12-11T22:41:00,210 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN in 388 msec 2024-12-11T22:41:00,211 INFO [StoreOpener-da518288db554a985096494f24adc59f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region da518288db554a985096494f24adc59f columnFamilyName family 2024-12-11T22:41:00,211 DEBUG [StoreOpener-da518288db554a985096494f24adc59f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:41:00,211 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for de3803c768bc063bdc71e495af58eb48 2024-12-11T22:41:00,212 INFO [StoreOpener-da518288db554a985096494f24adc59f-1 {}] regionserver.HStore(327): Store=da518288db554a985096494f24adc59f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:41:00,212 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for da518288db554a985096494f24adc59f 2024-12-11T22:41:00,213 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=9f169feb3190772cec56c0ae51d57c26, regionState=OPEN, openSeqNum=2, regionLocation=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:00,214 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/da518288db554a985096494f24adc59f 2024-12-11T22:41:00,216 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/da518288db554a985096494f24adc59f 2024-12-11T22:41:00,216 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/de3803c768bc063bdc71e495af58eb48/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T22:41:00,216 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9f169feb3190772cec56c0ae51d57c26, server=56d7a8405f4a,40421,1733956857131 because future has completed 2024-12-11T22:41:00,216 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for da518288db554a985096494f24adc59f 2024-12-11T22:41:00,216 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for da518288db554a985096494f24adc59f 2024-12-11T22:41:00,216 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened de3803c768bc063bdc71e495af58eb48; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67993005, jitterRate=0.01317472755908966}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:41:00,217 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for de3803c768bc063bdc71e495af58eb48 2024-12-11T22:41:00,217 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for de3803c768bc063bdc71e495af58eb48: Running coprocessor pre-open hook at 1733956860198Writing region info on filesystem at 1733956860198Initializing all the Stores at 1733956860200 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733956860200Cleaning up temporary data from old regions at 1733956860209 (+9 ms)Running coprocessor post-open hooks at 1733956860217 (+8 ms)Region opened successfully at 1733956860217 2024-12-11T22:41:00,218 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1733956859502.de3803c768bc063bdc71e495af58eb48., pid=43, masterSystemTime=1733956860167 2024-12-11T22:41:00,220 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for da518288db554a985096494f24adc59f 2024-12-11T22:41:00,221 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1733956859502.de3803c768bc063bdc71e495af58eb48. 2024-12-11T22:41:00,221 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,ggg,1733956859502.de3803c768bc063bdc71e495af58eb48. 2024-12-11T22:41:00,222 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,jjj,1733956859502.ad4bb3db7f3f05f24b6115fc887d6ab5. 2024-12-11T22:41:00,222 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => ad4bb3db7f3f05f24b6115fc887d6ab5, NAME => 'test,jjj,1733956859502.ad4bb3db7f3f05f24b6115fc887d6ab5.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-12-11T22:41:00,222 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ad4bb3db7f3f05f24b6115fc887d6ab5 2024-12-11T22:41:00,222 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,jjj,1733956859502.ad4bb3db7f3f05f24b6115fc887d6ab5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:41:00,223 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for ad4bb3db7f3f05f24b6115fc887d6ab5 2024-12-11T22:41:00,223 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for ad4bb3db7f3f05f24b6115fc887d6ab5 2024-12-11T22:41:00,223 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=8 2024-12-11T22:41:00,224 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 9f169feb3190772cec56c0ae51d57c26, server=56d7a8405f4a,40421,1733956857131 in 210 msec 2024-12-11T22:41:00,224 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/da518288db554a985096494f24adc59f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T22:41:00,225 INFO [StoreOpener-ad4bb3db7f3f05f24b6115fc887d6ab5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ad4bb3db7f3f05f24b6115fc887d6ab5 2024-12-11T22:41:00,226 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened da518288db554a985096494f24adc59f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73212983, jitterRate=0.09095846116542816}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:41:00,226 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for da518288db554a985096494f24adc59f 2024-12-11T22:41:00,226 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for da518288db554a985096494f24adc59f: Running coprocessor pre-open hook at 1733956860206Writing region info on filesystem at 1733956860206Initializing all the Stores at 1733956860207 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733956860207Cleaning up temporary data from old regions at 1733956860216 (+9 ms)Running coprocessor post-open hooks at 1733956860226 (+10 ms)Region opened successfully at 1733956860226 2024-12-11T22:41:00,227 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=9f169feb3190772cec56c0ae51d57c26, ASSIGN in 405 msec 2024-12-11T22:41:00,227 INFO [StoreOpener-ad4bb3db7f3f05f24b6115fc887d6ab5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ad4bb3db7f3f05f24b6115fc887d6ab5 columnFamilyName family 2024-12-11T22:41:00,227 DEBUG [StoreOpener-ad4bb3db7f3f05f24b6115fc887d6ab5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:41:00,227 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1733956859502.da518288db554a985096494f24adc59f., pid=39, masterSystemTime=1733956860148 2024-12-11T22:41:00,228 INFO [StoreOpener-ad4bb3db7f3f05f24b6115fc887d6ab5-1 {}] regionserver.HStore(327): Store=ad4bb3db7f3f05f24b6115fc887d6ab5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:41:00,228 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for ad4bb3db7f3f05f24b6115fc887d6ab5 2024-12-11T22:41:00,229 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/ad4bb3db7f3f05f24b6115fc887d6ab5 2024-12-11T22:41:00,229 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=de3803c768bc063bdc71e495af58eb48, regionState=OPEN, openSeqNum=2, regionLocation=56d7a8405f4a,36971,1733956857186 2024-12-11T22:41:00,230 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/ad4bb3db7f3f05f24b6115fc887d6ab5 2024-12-11T22:41:00,232 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1733956859502.da518288db554a985096494f24adc59f. 2024-12-11T22:41:00,232 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,ttt,1733956859502.da518288db554a985096494f24adc59f. 2024-12-11T22:41:00,232 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34843 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=56d7a8405f4a,36971,1733956857186, table=test, region=de3803c768bc063bdc71e495af58eb48. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-11T22:41:00,232 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,hhh,1733956859502.b31ee44b6609c355adc37986b13d239c. 2024-12-11T22:41:00,232 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for ad4bb3db7f3f05f24b6115fc887d6ab5 2024-12-11T22:41:00,233 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for ad4bb3db7f3f05f24b6115fc887d6ab5 2024-12-11T22:41:00,233 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => b31ee44b6609c355adc37986b13d239c, NAME => 'test,hhh,1733956859502.b31ee44b6609c355adc37986b13d239c.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-12-11T22:41:00,233 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure de3803c768bc063bdc71e495af58eb48, server=56d7a8405f4a,36971,1733956857186 because future has completed 2024-12-11T22:41:00,233 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b31ee44b6609c355adc37986b13d239c 2024-12-11T22:41:00,233 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,hhh,1733956859502.b31ee44b6609c355adc37986b13d239c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:41:00,233 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for b31ee44b6609c355adc37986b13d239c 2024-12-11T22:41:00,233 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for b31ee44b6609c355adc37986b13d239c 2024-12-11T22:41:00,234 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=da518288db554a985096494f24adc59f, regionState=OPEN, openSeqNum=2, regionLocation=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:00,236 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for ad4bb3db7f3f05f24b6115fc887d6ab5 2024-12-11T22:41:00,237 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-11T22:41:00,238 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure da518288db554a985096494f24adc59f, server=56d7a8405f4a,40421,1733956857131 because future has completed 2024-12-11T22:41:00,239 INFO [StoreOpener-b31ee44b6609c355adc37986b13d239c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b31ee44b6609c355adc37986b13d239c 2024-12-11T22:41:00,240 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52459, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-11T22:41:00,243 INFO [StoreOpener-b31ee44b6609c355adc37986b13d239c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b31ee44b6609c355adc37986b13d239c columnFamilyName family 2024-12-11T22:41:00,243 DEBUG [StoreOpener-b31ee44b6609c355adc37986b13d239c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:41:00,243 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=11 2024-12-11T22:41:00,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure de3803c768bc063bdc71e495af58eb48, server=56d7a8405f4a,36971,1733956857186 in 181 msec 2024-12-11T22:41:00,244 INFO [StoreOpener-b31ee44b6609c355adc37986b13d239c-1 {}] regionserver.HStore(327): Store=b31ee44b6609c355adc37986b13d239c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:41:00,244 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for b31ee44b6609c355adc37986b13d239c 2024-12-11T22:41:00,246 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/b31ee44b6609c355adc37986b13d239c 2024-12-11T22:41:00,246 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/ad4bb3db7f3f05f24b6115fc887d6ab5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T22:41:00,247 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,uuu,1733956859502.f7dbe11f45ae33a883424947e613c7c6. 2024-12-11T22:41:00,247 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened ad4bb3db7f3f05f24b6115fc887d6ab5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59505228, jitterRate=-0.11330300569534302}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:41:00,247 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ad4bb3db7f3f05f24b6115fc887d6ab5 2024-12-11T22:41:00,247 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => f7dbe11f45ae33a883424947e613c7c6, NAME => 'test,uuu,1733956859502.f7dbe11f45ae33a883424947e613c7c6.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-12-11T22:41:00,248 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/b31ee44b6609c355adc37986b13d239c 2024-12-11T22:41:00,248 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN in 425 msec 2024-12-11T22:41:00,248 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for ad4bb3db7f3f05f24b6115fc887d6ab5: Running coprocessor pre-open hook at 1733956860223Writing region info on filesystem at 1733956860223Initializing all the Stores at 1733956860224 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733956860224Cleaning up temporary data from old regions at 1733956860233 (+9 ms)Running coprocessor post-open hooks at 1733956860248 (+15 ms)Region opened successfully at 1733956860248 2024-12-11T22:41:00,248 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f7dbe11f45ae33a883424947e613c7c6 2024-12-11T22:41:00,248 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,uuu,1733956859502.f7dbe11f45ae33a883424947e613c7c6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:41:00,248 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for f7dbe11f45ae33a883424947e613c7c6 2024-12-11T22:41:00,248 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for f7dbe11f45ae33a883424947e613c7c6 2024-12-11T22:41:00,249 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1733956859502.ad4bb3db7f3f05f24b6115fc887d6ab5., pid=44, masterSystemTime=1733956860167 2024-12-11T22:41:00,249 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for b31ee44b6609c355adc37986b13d239c 2024-12-11T22:41:00,249 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for b31ee44b6609c355adc37986b13d239c 2024-12-11T22:41:00,251 INFO [StoreOpener-f7dbe11f45ae33a883424947e613c7c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f7dbe11f45ae33a883424947e613c7c6 2024-12-11T22:41:00,252 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1733956859502.ad4bb3db7f3f05f24b6115fc887d6ab5. 2024-12-11T22:41:00,252 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,jjj,1733956859502.ad4bb3db7f3f05f24b6115fc887d6ab5. 2024-12-11T22:41:00,252 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,xxx,1733956859502.a0e02a7a4f8d57169d1f435eabc00839. 2024-12-11T22:41:00,252 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => a0e02a7a4f8d57169d1f435eabc00839, NAME => 'test,xxx,1733956859502.a0e02a7a4f8d57169d1f435eabc00839.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-12-11T22:41:00,253 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a0e02a7a4f8d57169d1f435eabc00839 2024-12-11T22:41:00,253 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for b31ee44b6609c355adc37986b13d239c 2024-12-11T22:41:00,253 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,xxx,1733956859502.a0e02a7a4f8d57169d1f435eabc00839.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:41:00,253 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for a0e02a7a4f8d57169d1f435eabc00839 2024-12-11T22:41:00,254 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for a0e02a7a4f8d57169d1f435eabc00839 2024-12-11T22:41:00,256 INFO [StoreOpener-f7dbe11f45ae33a883424947e613c7c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7dbe11f45ae33a883424947e613c7c6 columnFamilyName family 2024-12-11T22:41:00,256 DEBUG [StoreOpener-f7dbe11f45ae33a883424947e613c7c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:41:00,256 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=24 2024-12-11T22:41:00,256 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure da518288db554a985096494f24adc59f, server=56d7a8405f4a,40421,1733956857131 in 214 msec 2024-12-11T22:41:00,256 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/b31ee44b6609c355adc37986b13d239c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T22:41:00,256 INFO [StoreOpener-f7dbe11f45ae33a883424947e613c7c6-1 {}] regionserver.HStore(327): Store=f7dbe11f45ae33a883424947e613c7c6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:41:00,257 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for f7dbe11f45ae33a883424947e613c7c6 2024-12-11T22:41:00,257 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened b31ee44b6609c355adc37986b13d239c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61200357, jitterRate=-0.08804361522197723}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:41:00,258 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b31ee44b6609c355adc37986b13d239c 2024-12-11T22:41:00,258 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/f7dbe11f45ae33a883424947e613c7c6 2024-12-11T22:41:00,258 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for b31ee44b6609c355adc37986b13d239c: Running coprocessor pre-open hook at 1733956860233Writing region info on filesystem at 1733956860233Initializing all the Stores at 1733956860236 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733956860236Cleaning up temporary data from old regions at 1733956860249 (+13 ms)Running coprocessor post-open hooks at 1733956860258 (+9 ms)Region opened successfully at 1733956860258 2024-12-11T22:41:00,259 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/f7dbe11f45ae33a883424947e613c7c6 2024-12-11T22:41:00,259 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1733956859502.b31ee44b6609c355adc37986b13d239c., pid=36, masterSystemTime=1733956860148 2024-12-11T22:41:00,259 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=da518288db554a985096494f24adc59f, ASSIGN in 437 msec 2024-12-11T22:41:00,259 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for f7dbe11f45ae33a883424947e613c7c6 2024-12-11T22:41:00,259 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for f7dbe11f45ae33a883424947e613c7c6 2024-12-11T22:41:00,261 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=ad4bb3db7f3f05f24b6115fc887d6ab5, regionState=OPEN, openSeqNum=2, regionLocation=56d7a8405f4a,36971,1733956857186 2024-12-11T22:41:00,262 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1733956859502.b31ee44b6609c355adc37986b13d239c. 2024-12-11T22:41:00,262 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,hhh,1733956859502.b31ee44b6609c355adc37986b13d239c. 2024-12-11T22:41:00,262 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,lll,1733956859502.445df7fd625f94e0eb0712fcd777ebda. 2024-12-11T22:41:00,263 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 445df7fd625f94e0eb0712fcd777ebda, NAME => 'test,lll,1733956859502.445df7fd625f94e0eb0712fcd777ebda.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-12-11T22:41:00,263 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 445df7fd625f94e0eb0712fcd777ebda 2024-12-11T22:41:00,263 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,lll,1733956859502.445df7fd625f94e0eb0712fcd777ebda.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:41:00,264 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 445df7fd625f94e0eb0712fcd777ebda 2024-12-11T22:41:00,264 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 445df7fd625f94e0eb0712fcd777ebda 2024-12-11T22:41:00,264 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for f7dbe11f45ae33a883424947e613c7c6 2024-12-11T22:41:00,264 INFO [StoreOpener-a0e02a7a4f8d57169d1f435eabc00839-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a0e02a7a4f8d57169d1f435eabc00839 2024-12-11T22:41:00,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure ad4bb3db7f3f05f24b6115fc887d6ab5, server=56d7a8405f4a,36971,1733956857186 because future has completed 2024-12-11T22:41:00,267 INFO [StoreOpener-445df7fd625f94e0eb0712fcd777ebda-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 445df7fd625f94e0eb0712fcd777ebda 2024-12-11T22:41:00,268 INFO [StoreOpener-a0e02a7a4f8d57169d1f435eabc00839-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a0e02a7a4f8d57169d1f435eabc00839 columnFamilyName family 2024-12-11T22:41:00,268 DEBUG [StoreOpener-a0e02a7a4f8d57169d1f435eabc00839-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:41:00,268 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/f7dbe11f45ae33a883424947e613c7c6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T22:41:00,269 INFO [StoreOpener-a0e02a7a4f8d57169d1f435eabc00839-1 {}] regionserver.HStore(327): Store=a0e02a7a4f8d57169d1f435eabc00839/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:41:00,270 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for a0e02a7a4f8d57169d1f435eabc00839 2024-12-11T22:41:00,270 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened f7dbe11f45ae33a883424947e613c7c6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61437998, jitterRate=-0.08450248837471008}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:41:00,270 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f7dbe11f45ae33a883424947e613c7c6 2024-12-11T22:41:00,271 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for f7dbe11f45ae33a883424947e613c7c6: Running coprocessor pre-open hook at 1733956860248Writing region info on filesystem at 1733956860248Initializing all the Stores at 1733956860250 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733956860250Cleaning up temporary data from old regions at 1733956860260 (+10 ms)Running coprocessor post-open hooks at 1733956860270 (+10 ms)Region opened successfully at 1733956860270 2024-12-11T22:41:00,271 INFO [StoreOpener-445df7fd625f94e0eb0712fcd777ebda-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 445df7fd625f94e0eb0712fcd777ebda columnFamilyName family 2024-12-11T22:41:00,271 DEBUG [StoreOpener-445df7fd625f94e0eb0712fcd777ebda-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:41:00,272 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/a0e02a7a4f8d57169d1f435eabc00839 2024-12-11T22:41:00,272 INFO [StoreOpener-445df7fd625f94e0eb0712fcd777ebda-1 {}] regionserver.HStore(327): Store=445df7fd625f94e0eb0712fcd777ebda/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:41:00,272 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 445df7fd625f94e0eb0712fcd777ebda 2024-12-11T22:41:00,272 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/a0e02a7a4f8d57169d1f435eabc00839 2024-12-11T22:41:00,273 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1733956859502.f7dbe11f45ae33a883424947e613c7c6., pid=48, masterSystemTime=1733956860237 2024-12-11T22:41:00,274 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for a0e02a7a4f8d57169d1f435eabc00839 2024-12-11T22:41:00,274 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for a0e02a7a4f8d57169d1f435eabc00839 2024-12-11T22:41:00,274 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/445df7fd625f94e0eb0712fcd777ebda 2024-12-11T22:41:00,275 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=14 2024-12-11T22:41:00,275 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/445df7fd625f94e0eb0712fcd777ebda 2024-12-11T22:41:00,275 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure ad4bb3db7f3f05f24b6115fc887d6ab5, server=56d7a8405f4a,36971,1733956857186 in 207 msec 2024-12-11T22:41:00,275 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 445df7fd625f94e0eb0712fcd777ebda 2024-12-11T22:41:00,276 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 445df7fd625f94e0eb0712fcd777ebda 2024-12-11T22:41:00,276 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1733956859502.f7dbe11f45ae33a883424947e613c7c6. 2024-12-11T22:41:00,276 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,uuu,1733956859502.f7dbe11f45ae33a883424947e613c7c6. 2024-12-11T22:41:00,276 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,ooo,1733956859502.b1a38f11b48b53983535b863fd4392ff. 2024-12-11T22:41:00,277 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => b1a38f11b48b53983535b863fd4392ff, NAME => 'test,ooo,1733956859502.b1a38f11b48b53983535b863fd4392ff.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-12-11T22:41:00,277 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b1a38f11b48b53983535b863fd4392ff 2024-12-11T22:41:00,277 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,ooo,1733956859502.b1a38f11b48b53983535b863fd4392ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:41:00,277 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for b1a38f11b48b53983535b863fd4392ff 2024-12-11T22:41:00,278 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for b1a38f11b48b53983535b863fd4392ff 2024-12-11T22:41:00,279 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for a0e02a7a4f8d57169d1f435eabc00839 2024-12-11T22:41:00,279 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN in 456 msec 2024-12-11T22:41:00,280 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 445df7fd625f94e0eb0712fcd777ebda 2024-12-11T22:41:00,281 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=f7dbe11f45ae33a883424947e613c7c6, regionState=OPEN, openSeqNum=2, regionLocation=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:00,284 INFO [StoreOpener-b1a38f11b48b53983535b863fd4392ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b1a38f11b48b53983535b863fd4392ff 2024-12-11T22:41:00,285 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/a0e02a7a4f8d57169d1f435eabc00839/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T22:41:00,286 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/445df7fd625f94e0eb0712fcd777ebda/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T22:41:00,286 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure f7dbe11f45ae33a883424947e613c7c6, server=56d7a8405f4a,34703,1733956857011 because future has completed 2024-12-11T22:41:00,286 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened a0e02a7a4f8d57169d1f435eabc00839; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60576656, jitterRate=-0.09733748435974121}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:41:00,286 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 445df7fd625f94e0eb0712fcd777ebda; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60122849, jitterRate=-0.10409973561763763}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:41:00,286 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a0e02a7a4f8d57169d1f435eabc00839 2024-12-11T22:41:00,286 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 445df7fd625f94e0eb0712fcd777ebda 2024-12-11T22:41:00,286 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 445df7fd625f94e0eb0712fcd777ebda: Running coprocessor pre-open hook at 1733956860264Writing region info on filesystem at 1733956860264Initializing all the Stores at 1733956860267 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733956860267Cleaning up temporary data from old regions at 1733956860276 (+9 ms)Running coprocessor post-open hooks at 1733956860286 (+10 ms)Region opened successfully at 1733956860286 2024-12-11T22:41:00,286 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for a0e02a7a4f8d57169d1f435eabc00839: Running coprocessor pre-open hook at 1733956860254Writing region info on filesystem at 1733956860254Initializing all the Stores at 1733956860255 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733956860255Cleaning up temporary data from old regions at 1733956860274 (+19 ms)Running coprocessor post-open hooks at 1733956860286 (+12 ms)Region opened successfully at 1733956860286 2024-12-11T22:41:00,287 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=b31ee44b6609c355adc37986b13d239c, regionState=OPEN, openSeqNum=2, regionLocation=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:00,288 INFO [StoreOpener-b1a38f11b48b53983535b863fd4392ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b1a38f11b48b53983535b863fd4392ff columnFamilyName family 2024-12-11T22:41:00,288 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1733956859502.a0e02a7a4f8d57169d1f435eabc00839., pid=34, masterSystemTime=1733956860167 2024-12-11T22:41:00,288 DEBUG [StoreOpener-b1a38f11b48b53983535b863fd4392ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:41:00,288 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1733956859502.445df7fd625f94e0eb0712fcd777ebda., pid=37, masterSystemTime=1733956860148 2024-12-11T22:41:00,290 INFO [StoreOpener-b1a38f11b48b53983535b863fd4392ff-1 {}] regionserver.HStore(327): Store=b1a38f11b48b53983535b863fd4392ff/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:41:00,290 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for b1a38f11b48b53983535b863fd4392ff 2024-12-11T22:41:00,291 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure b31ee44b6609c355adc37986b13d239c, server=56d7a8405f4a,40421,1733956857131 because future has completed 2024-12-11T22:41:00,291 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1733956859502.a0e02a7a4f8d57169d1f435eabc00839. 2024-12-11T22:41:00,291 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,xxx,1733956859502.a0e02a7a4f8d57169d1f435eabc00839. 2024-12-11T22:41:00,291 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,mmm,1733956859502.f666d29f964372d8719d169fcecef641. 2024-12-11T22:41:00,292 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => f666d29f964372d8719d169fcecef641, NAME => 'test,mmm,1733956859502.f666d29f964372d8719d169fcecef641.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-12-11T22:41:00,292 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f666d29f964372d8719d169fcecef641 2024-12-11T22:41:00,292 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,mmm,1733956859502.f666d29f964372d8719d169fcecef641.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:41:00,292 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for f666d29f964372d8719d169fcecef641 2024-12-11T22:41:00,292 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for f666d29f964372d8719d169fcecef641 2024-12-11T22:41:00,293 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1733956859502.445df7fd625f94e0eb0712fcd777ebda. 2024-12-11T22:41:00,293 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,lll,1733956859502.445df7fd625f94e0eb0712fcd777ebda. 2024-12-11T22:41:00,293 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,bbb,1733956859502.66aac9e74e8263b0b8914fdc86eec66c. 2024-12-11T22:41:00,293 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 66aac9e74e8263b0b8914fdc86eec66c, NAME => 'test,bbb,1733956859502.66aac9e74e8263b0b8914fdc86eec66c.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-12-11T22:41:00,294 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 66aac9e74e8263b0b8914fdc86eec66c 2024-12-11T22:41:00,294 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,bbb,1733956859502.66aac9e74e8263b0b8914fdc86eec66c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:41:00,294 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 66aac9e74e8263b0b8914fdc86eec66c 2024-12-11T22:41:00,294 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/b1a38f11b48b53983535b863fd4392ff 2024-12-11T22:41:00,294 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 66aac9e74e8263b0b8914fdc86eec66c 2024-12-11T22:41:00,295 INFO [StoreOpener-f666d29f964372d8719d169fcecef641-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f666d29f964372d8719d169fcecef641 2024-12-11T22:41:00,295 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/b1a38f11b48b53983535b863fd4392ff 2024-12-11T22:41:00,297 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for b1a38f11b48b53983535b863fd4392ff 2024-12-11T22:41:00,297 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for b1a38f11b48b53983535b863fd4392ff 2024-12-11T22:41:00,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=25 2024-12-11T22:41:00,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure f7dbe11f45ae33a883424947e613c7c6, server=56d7a8405f4a,34703,1733956857011 in 208 msec 2024-12-11T22:41:00,298 INFO [StoreOpener-f666d29f964372d8719d169fcecef641-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f666d29f964372d8719d169fcecef641 columnFamilyName family 2024-12-11T22:41:00,298 DEBUG [StoreOpener-f666d29f964372d8719d169fcecef641-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:41:00,299 INFO [StoreOpener-f666d29f964372d8719d169fcecef641-1 {}] regionserver.HStore(327): Store=f666d29f964372d8719d169fcecef641/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:41:00,299 INFO [StoreOpener-66aac9e74e8263b0b8914fdc86eec66c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 66aac9e74e8263b0b8914fdc86eec66c 2024-12-11T22:41:00,299 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for f666d29f964372d8719d169fcecef641 2024-12-11T22:41:00,299 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for b1a38f11b48b53983535b863fd4392ff 2024-12-11T22:41:00,300 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/f666d29f964372d8719d169fcecef641 2024-12-11T22:41:00,301 INFO [StoreOpener-66aac9e74e8263b0b8914fdc86eec66c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66aac9e74e8263b0b8914fdc86eec66c columnFamilyName family 2024-12-11T22:41:00,301 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/f666d29f964372d8719d169fcecef641 2024-12-11T22:41:00,301 DEBUG [StoreOpener-66aac9e74e8263b0b8914fdc86eec66c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:41:00,302 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for f666d29f964372d8719d169fcecef641 2024-12-11T22:41:00,302 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for f666d29f964372d8719d169fcecef641 2024-12-11T22:41:00,302 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f7dbe11f45ae33a883424947e613c7c6, ASSIGN in 478 msec 2024-12-11T22:41:00,302 INFO [StoreOpener-66aac9e74e8263b0b8914fdc86eec66c-1 {}] regionserver.HStore(327): Store=66aac9e74e8263b0b8914fdc86eec66c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:41:00,302 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 66aac9e74e8263b0b8914fdc86eec66c 2024-12-11T22:41:00,304 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=445df7fd625f94e0eb0712fcd777ebda, regionState=OPEN, openSeqNum=2, regionLocation=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:00,304 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/66aac9e74e8263b0b8914fdc86eec66c 2024-12-11T22:41:00,304 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/b1a38f11b48b53983535b863fd4392ff/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T22:41:00,305 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/66aac9e74e8263b0b8914fdc86eec66c 2024-12-11T22:41:00,305 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for f666d29f964372d8719d169fcecef641 2024-12-11T22:41:00,306 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened b1a38f11b48b53983535b863fd4392ff; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61083869, jitterRate=-0.08977942168712616}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:41:00,306 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 66aac9e74e8263b0b8914fdc86eec66c 2024-12-11T22:41:00,306 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b1a38f11b48b53983535b863fd4392ff 2024-12-11T22:41:00,307 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 66aac9e74e8263b0b8914fdc86eec66c 2024-12-11T22:41:00,307 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for b1a38f11b48b53983535b863fd4392ff: Running coprocessor pre-open hook at 1733956860278Writing region info on filesystem at 1733956860278Initializing all the Stores at 1733956860279 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733956860280 (+1 ms)Cleaning up temporary data from old regions at 1733956860297 (+17 ms)Running coprocessor post-open hooks at 1733956860307 (+10 ms)Region opened successfully at 1733956860307 2024-12-11T22:41:00,308 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 445df7fd625f94e0eb0712fcd777ebda, server=56d7a8405f4a,40421,1733956857131 because future has completed 2024-12-11T22:41:00,309 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1733956859502.b1a38f11b48b53983535b863fd4392ff., pid=49, masterSystemTime=1733956860237 2024-12-11T22:41:00,312 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 66aac9e74e8263b0b8914fdc86eec66c 2024-12-11T22:41:00,312 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/f666d29f964372d8719d169fcecef641/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T22:41:00,313 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened f666d29f964372d8719d169fcecef641; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59341878, jitterRate=-0.11573711037635803}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:41:00,313 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f666d29f964372d8719d169fcecef641 2024-12-11T22:41:00,313 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1733956859502.b1a38f11b48b53983535b863fd4392ff. 2024-12-11T22:41:00,314 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,ooo,1733956859502.b1a38f11b48b53983535b863fd4392ff. 2024-12-11T22:41:00,314 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for f666d29f964372d8719d169fcecef641: Running coprocessor pre-open hook at 1733956860293Writing region info on filesystem at 1733956860293Initializing all the Stores at 1733956860295 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733956860295Cleaning up temporary data from old regions at 1733956860302 (+7 ms)Running coprocessor post-open hooks at 1733956860313 (+11 ms)Region opened successfully at 1733956860314 (+1 ms) 2024-12-11T22:41:00,314 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,sss,1733956859502.daa100e1546a41747fac83159daee190. 2024-12-11T22:41:00,314 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => daa100e1546a41747fac83159daee190, NAME => 'test,sss,1733956859502.daa100e1546a41747fac83159daee190.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-12-11T22:41:00,314 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test daa100e1546a41747fac83159daee190 2024-12-11T22:41:00,315 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,sss,1733956859502.daa100e1546a41747fac83159daee190.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:41:00,315 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for daa100e1546a41747fac83159daee190 2024-12-11T22:41:00,315 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for daa100e1546a41747fac83159daee190 2024-12-11T22:41:00,316 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1733956859502.f666d29f964372d8719d169fcecef641., pid=42, masterSystemTime=1733956860167 2024-12-11T22:41:00,316 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=12 2024-12-11T22:41:00,316 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure b31ee44b6609c355adc37986b13d239c, server=56d7a8405f4a,40421,1733956857131 in 286 msec 2024-12-11T22:41:00,317 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/66aac9e74e8263b0b8914fdc86eec66c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T22:41:00,318 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 66aac9e74e8263b0b8914fdc86eec66c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66094894, jitterRate=-0.015109330415725708}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:41:00,318 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 66aac9e74e8263b0b8914fdc86eec66c 2024-12-11T22:41:00,318 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 66aac9e74e8263b0b8914fdc86eec66c: Running coprocessor pre-open hook at 1733956860294Writing region info on filesystem at 1733956860294Initializing all the Stores at 1733956860295 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733956860296 (+1 ms)Cleaning up temporary data from old regions at 1733956860307 (+11 ms)Running coprocessor post-open hooks at 1733956860318 (+11 ms)Region opened successfully at 1733956860318 2024-12-11T22:41:00,319 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1733956859502.66aac9e74e8263b0b8914fdc86eec66c., pid=35, masterSystemTime=1733956860148 2024-12-11T22:41:00,319 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1733956859502.f666d29f964372d8719d169fcecef641. 2024-12-11T22:41:00,319 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,mmm,1733956859502.f666d29f964372d8719d169fcecef641. 2024-12-11T22:41:00,320 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,rrr,1733956859502.f219e06826a4a3ede110adacdf28f6f4. 2024-12-11T22:41:00,320 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => f219e06826a4a3ede110adacdf28f6f4, NAME => 'test,rrr,1733956859502.f219e06826a4a3ede110adacdf28f6f4.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-12-11T22:41:00,320 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f219e06826a4a3ede110adacdf28f6f4 2024-12-11T22:41:00,320 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,rrr,1733956859502.f219e06826a4a3ede110adacdf28f6f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:41:00,320 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for f219e06826a4a3ede110adacdf28f6f4 2024-12-11T22:41:00,321 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for f219e06826a4a3ede110adacdf28f6f4 2024-12-11T22:41:00,323 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b31ee44b6609c355adc37986b13d239c, ASSIGN in 498 msec 2024-12-11T22:41:00,325 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=66aac9e74e8263b0b8914fdc86eec66c, regionState=OPEN, openSeqNum=2, regionLocation=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:00,326 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1733956859502.66aac9e74e8263b0b8914fdc86eec66c. 2024-12-11T22:41:00,326 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,bbb,1733956859502.66aac9e74e8263b0b8914fdc86eec66c. 2024-12-11T22:41:00,326 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34843 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=56d7a8405f4a,40421,1733956857131, table=test, region=445df7fd625f94e0eb0712fcd777ebda. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-11T22:41:00,326 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,qqq,1733956859502.2f551d316d038ca1e1403bd05f890572. 2024-12-11T22:41:00,327 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 2f551d316d038ca1e1403bd05f890572, NAME => 'test,qqq,1733956859502.2f551d316d038ca1e1403bd05f890572.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-12-11T22:41:00,327 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2f551d316d038ca1e1403bd05f890572 2024-12-11T22:41:00,327 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,qqq,1733956859502.2f551d316d038ca1e1403bd05f890572.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:41:00,327 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 2f551d316d038ca1e1403bd05f890572 2024-12-11T22:41:00,328 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 2f551d316d038ca1e1403bd05f890572 2024-12-11T22:41:00,328 INFO [StoreOpener-daa100e1546a41747fac83159daee190-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region daa100e1546a41747fac83159daee190 2024-12-11T22:41:00,328 INFO [StoreOpener-f219e06826a4a3ede110adacdf28f6f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f219e06826a4a3ede110adacdf28f6f4 2024-12-11T22:41:00,329 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66aac9e74e8263b0b8914fdc86eec66c, server=56d7a8405f4a,40421,1733956857131 because future has completed 2024-12-11T22:41:00,330 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=f666d29f964372d8719d169fcecef641, regionState=OPEN, openSeqNum=2, regionLocation=56d7a8405f4a,36971,1733956857186 2024-12-11T22:41:00,330 INFO [StoreOpener-f219e06826a4a3ede110adacdf28f6f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f219e06826a4a3ede110adacdf28f6f4 columnFamilyName family 2024-12-11T22:41:00,330 DEBUG [StoreOpener-f219e06826a4a3ede110adacdf28f6f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:41:00,330 INFO [StoreOpener-2f551d316d038ca1e1403bd05f890572-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2f551d316d038ca1e1403bd05f890572 2024-12-11T22:41:00,330 INFO [StoreOpener-daa100e1546a41747fac83159daee190-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region daa100e1546a41747fac83159daee190 columnFamilyName family 2024-12-11T22:41:00,330 DEBUG [StoreOpener-daa100e1546a41747fac83159daee190-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:41:00,331 INFO [StoreOpener-f219e06826a4a3ede110adacdf28f6f4-1 {}] regionserver.HStore(327): Store=f219e06826a4a3ede110adacdf28f6f4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:41:00,331 INFO [StoreOpener-daa100e1546a41747fac83159daee190-1 {}] regionserver.HStore(327): Store=daa100e1546a41747fac83159daee190/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:41:00,332 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for f219e06826a4a3ede110adacdf28f6f4 2024-12-11T22:41:00,331 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for daa100e1546a41747fac83159daee190 2024-12-11T22:41:00,333 INFO [StoreOpener-2f551d316d038ca1e1403bd05f890572-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2f551d316d038ca1e1403bd05f890572 columnFamilyName family 2024-12-11T22:41:00,333 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/daa100e1546a41747fac83159daee190 2024-12-11T22:41:00,333 DEBUG [StoreOpener-2f551d316d038ca1e1403bd05f890572-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:41:00,333 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/f219e06826a4a3ede110adacdf28f6f4 2024-12-11T22:41:00,334 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure f666d29f964372d8719d169fcecef641, server=56d7a8405f4a,36971,1733956857186 because future has completed 2024-12-11T22:41:00,335 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/f219e06826a4a3ede110adacdf28f6f4 2024-12-11T22:41:00,335 INFO [StoreOpener-2f551d316d038ca1e1403bd05f890572-1 {}] regionserver.HStore(327): Store=2f551d316d038ca1e1403bd05f890572/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:41:00,335 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 2f551d316d038ca1e1403bd05f890572 2024-12-11T22:41:00,336 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for f219e06826a4a3ede110adacdf28f6f4 2024-12-11T22:41:00,336 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for f219e06826a4a3ede110adacdf28f6f4 2024-12-11T22:41:00,336 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/daa100e1546a41747fac83159daee190 2024-12-11T22:41:00,336 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34843 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=56d7a8405f4a,36971,1733956857186, table=test, region=f666d29f964372d8719d169fcecef641. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-11T22:41:00,337 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/2f551d316d038ca1e1403bd05f890572 2024-12-11T22:41:00,338 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/2f551d316d038ca1e1403bd05f890572 2024-12-11T22:41:00,338 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for daa100e1546a41747fac83159daee190 2024-12-11T22:41:00,338 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for daa100e1546a41747fac83159daee190 2024-12-11T22:41:00,338 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 2f551d316d038ca1e1403bd05f890572 2024-12-11T22:41:00,338 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 2f551d316d038ca1e1403bd05f890572 2024-12-11T22:41:00,338 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for f219e06826a4a3ede110adacdf28f6f4 2024-12-11T22:41:00,342 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 2f551d316d038ca1e1403bd05f890572 2024-12-11T22:41:00,342 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for daa100e1546a41747fac83159daee190 2024-12-11T22:41:00,342 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=6 2024-12-11T22:41:00,342 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 66aac9e74e8263b0b8914fdc86eec66c, server=56d7a8405f4a,40421,1733956857131 in 318 msec 2024-12-11T22:41:00,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=66aac9e74e8263b0b8914fdc86eec66c, ASSIGN in 523 msec 2024-12-11T22:41:00,348 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/f219e06826a4a3ede110adacdf28f6f4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T22:41:00,348 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/2f551d316d038ca1e1403bd05f890572/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T22:41:00,348 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/daa100e1546a41747fac83159daee190/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T22:41:00,349 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 2f551d316d038ca1e1403bd05f890572; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58938762, jitterRate=-0.12174400687217712}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:41:00,349 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2f551d316d038ca1e1403bd05f890572 2024-12-11T22:41:00,349 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened f219e06826a4a3ede110adacdf28f6f4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74020049, jitterRate=0.10298468172550201}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:41:00,349 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f219e06826a4a3ede110adacdf28f6f4 2024-12-11T22:41:00,349 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 2f551d316d038ca1e1403bd05f890572: Running coprocessor pre-open hook at 1733956860328Writing region info on filesystem at 1733956860328Initializing all the Stores at 1733956860330 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733956860330Cleaning up temporary data from old regions at 1733956860338 (+8 ms)Running coprocessor post-open hooks at 1733956860349 (+11 ms)Region opened successfully at 1733956860349 2024-12-11T22:41:00,349 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for f219e06826a4a3ede110adacdf28f6f4: Running coprocessor pre-open hook at 1733956860321Writing region info on filesystem at 1733956860321Initializing all the Stores at 1733956860323 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733956860323Cleaning up temporary data from old regions at 1733956860336 (+13 ms)Running coprocessor post-open hooks at 1733956860349 (+13 ms)Region opened successfully at 1733956860349 2024-12-11T22:41:00,350 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1733956859502.2f551d316d038ca1e1403bd05f890572., pid=32, masterSystemTime=1733956860148 2024-12-11T22:41:00,351 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened daa100e1546a41747fac83159daee190; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75462330, jitterRate=0.1244763433933258}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:41:00,351 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for daa100e1546a41747fac83159daee190 2024-12-11T22:41:00,351 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1733956859502.f219e06826a4a3ede110adacdf28f6f4., pid=46, masterSystemTime=1733956860167 2024-12-11T22:41:00,352 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=17 2024-12-11T22:41:00,352 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure f666d29f964372d8719d169fcecef641, server=56d7a8405f4a,36971,1733956857186 in 293 msec 2024-12-11T22:41:00,352 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for daa100e1546a41747fac83159daee190: Running coprocessor pre-open hook at 1733956860315Writing region info on filesystem at 1733956860315Initializing all the Stores at 1733956860318 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733956860318Cleaning up temporary data from old regions at 1733956860338 (+20 ms)Running coprocessor post-open hooks at 1733956860351 (+13 ms)Region opened successfully at 1733956860352 (+1 ms) 2024-12-11T22:41:00,354 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1733956859502.daa100e1546a41747fac83159daee190., pid=50, masterSystemTime=1733956860237 2024-12-11T22:41:00,355 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1733956859502.2f551d316d038ca1e1403bd05f890572. 2024-12-11T22:41:00,355 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,qqq,1733956859502.2f551d316d038ca1e1403bd05f890572. 2024-12-11T22:41:00,355 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,nnn,1733956859502.65f30d8a32c00877e2cf6f22e7f50620. 2024-12-11T22:41:00,355 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 65f30d8a32c00877e2cf6f22e7f50620, NAME => 'test,nnn,1733956859502.65f30d8a32c00877e2cf6f22e7f50620.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-12-11T22:41:00,356 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 65f30d8a32c00877e2cf6f22e7f50620 2024-12-11T22:41:00,356 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,nnn,1733956859502.65f30d8a32c00877e2cf6f22e7f50620.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:41:00,356 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 65f30d8a32c00877e2cf6f22e7f50620 2024-12-11T22:41:00,356 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 65f30d8a32c00877e2cf6f22e7f50620 2024-12-11T22:41:00,357 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f666d29f964372d8719d169fcecef641, ASSIGN in 533 msec 2024-12-11T22:41:00,357 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1733956859502.f219e06826a4a3ede110adacdf28f6f4. 2024-12-11T22:41:00,357 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,rrr,1733956859502.f219e06826a4a3ede110adacdf28f6f4. 2024-12-11T22:41:00,358 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,,1733956859502.5a93ec512c97cf46e8942b91f4c8f088. 2024-12-11T22:41:00,358 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => 5a93ec512c97cf46e8942b91f4c8f088, NAME => 'test,,1733956859502.5a93ec512c97cf46e8942b91f4c8f088.', STARTKEY => '', ENDKEY => 'bbb'} 2024-12-11T22:41:00,359 INFO [StoreOpener-65f30d8a32c00877e2cf6f22e7f50620-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 65f30d8a32c00877e2cf6f22e7f50620 2024-12-11T22:41:00,359 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5a93ec512c97cf46e8942b91f4c8f088 2024-12-11T22:41:00,359 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,,1733956859502.5a93ec512c97cf46e8942b91f4c8f088.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:41:00,359 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for 5a93ec512c97cf46e8942b91f4c8f088 2024-12-11T22:41:00,359 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for 5a93ec512c97cf46e8942b91f4c8f088 2024-12-11T22:41:00,359 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1733956859502.daa100e1546a41747fac83159daee190. 2024-12-11T22:41:00,359 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=f219e06826a4a3ede110adacdf28f6f4, regionState=OPEN, openSeqNum=2, regionLocation=56d7a8405f4a,36971,1733956857186 2024-12-11T22:41:00,359 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,sss,1733956859502.daa100e1546a41747fac83159daee190. 2024-12-11T22:41:00,360 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,iii,1733956859502.00fd1046c94e7f507c0ee43e6555d034. 2024-12-11T22:41:00,360 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => 00fd1046c94e7f507c0ee43e6555d034, NAME => 'test,iii,1733956859502.00fd1046c94e7f507c0ee43e6555d034.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-12-11T22:41:00,360 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 00fd1046c94e7f507c0ee43e6555d034 2024-12-11T22:41:00,360 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,iii,1733956859502.00fd1046c94e7f507c0ee43e6555d034.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:41:00,361 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for 00fd1046c94e7f507c0ee43e6555d034 2024-12-11T22:41:00,361 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for 00fd1046c94e7f507c0ee43e6555d034 2024-12-11T22:41:00,363 INFO [StoreOpener-65f30d8a32c00877e2cf6f22e7f50620-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 65f30d8a32c00877e2cf6f22e7f50620 columnFamilyName family 2024-12-11T22:41:00,363 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure f219e06826a4a3ede110adacdf28f6f4, server=56d7a8405f4a,36971,1733956857186 because future has completed 2024-12-11T22:41:00,363 DEBUG [StoreOpener-65f30d8a32c00877e2cf6f22e7f50620-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:41:00,364 INFO [StoreOpener-65f30d8a32c00877e2cf6f22e7f50620-1 {}] regionserver.HStore(327): Store=65f30d8a32c00877e2cf6f22e7f50620/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:41:00,364 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 65f30d8a32c00877e2cf6f22e7f50620 2024-12-11T22:41:00,365 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=daa100e1546a41747fac83159daee190, regionState=OPEN, openSeqNum=2, regionLocation=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:00,365 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/65f30d8a32c00877e2cf6f22e7f50620 2024-12-11T22:41:00,365 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/65f30d8a32c00877e2cf6f22e7f50620 2024-12-11T22:41:00,366 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 65f30d8a32c00877e2cf6f22e7f50620 2024-12-11T22:41:00,366 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 65f30d8a32c00877e2cf6f22e7f50620 2024-12-11T22:41:00,368 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure daa100e1546a41747fac83159daee190, server=56d7a8405f4a,34703,1733956857011 because future has completed 2024-12-11T22:41:00,370 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 65f30d8a32c00877e2cf6f22e7f50620 2024-12-11T22:41:00,371 INFO [StoreOpener-5a93ec512c97cf46e8942b91f4c8f088-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5a93ec512c97cf46e8942b91f4c8f088 2024-12-11T22:41:00,373 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=22 2024-12-11T22:41:00,373 INFO [StoreOpener-5a93ec512c97cf46e8942b91f4c8f088-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a93ec512c97cf46e8942b91f4c8f088 columnFamilyName family 2024-12-11T22:41:00,373 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure f219e06826a4a3ede110adacdf28f6f4, server=56d7a8405f4a,36971,1733956857186 in 298 msec 2024-12-11T22:41:00,373 DEBUG [StoreOpener-5a93ec512c97cf46e8942b91f4c8f088-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:41:00,374 INFO [StoreOpener-5a93ec512c97cf46e8942b91f4c8f088-1 {}] regionserver.HStore(327): Store=5a93ec512c97cf46e8942b91f4c8f088/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:41:00,374 INFO [StoreOpener-00fd1046c94e7f507c0ee43e6555d034-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 00fd1046c94e7f507c0ee43e6555d034 2024-12-11T22:41:00,374 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for 5a93ec512c97cf46e8942b91f4c8f088 2024-12-11T22:41:00,375 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/65f30d8a32c00877e2cf6f22e7f50620/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T22:41:00,376 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/5a93ec512c97cf46e8942b91f4c8f088 2024-12-11T22:41:00,376 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 65f30d8a32c00877e2cf6f22e7f50620; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72915243, jitterRate=0.08652178943157196}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:41:00,376 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 65f30d8a32c00877e2cf6f22e7f50620 2024-12-11T22:41:00,376 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 65f30d8a32c00877e2cf6f22e7f50620: Running coprocessor pre-open hook at 1733956860356Writing region info on filesystem at 1733956860356Initializing all the Stores at 1733956860358 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733956860358Cleaning up temporary data from old regions at 1733956860366 (+8 ms)Running coprocessor post-open hooks at 1733956860376 (+10 ms)Region opened successfully at 1733956860376 2024-12-11T22:41:00,376 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/5a93ec512c97cf46e8942b91f4c8f088 2024-12-11T22:41:00,377 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN in 554 msec 2024-12-11T22:41:00,377 INFO [StoreOpener-00fd1046c94e7f507c0ee43e6555d034-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00fd1046c94e7f507c0ee43e6555d034 columnFamilyName family 2024-12-11T22:41:00,377 DEBUG [StoreOpener-00fd1046c94e7f507c0ee43e6555d034-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:41:00,377 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for 5a93ec512c97cf46e8942b91f4c8f088 2024-12-11T22:41:00,377 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for 5a93ec512c97cf46e8942b91f4c8f088 2024-12-11T22:41:00,378 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1733956859502.65f30d8a32c00877e2cf6f22e7f50620., pid=38, masterSystemTime=1733956860148 2024-12-11T22:41:00,378 INFO [StoreOpener-00fd1046c94e7f507c0ee43e6555d034-1 {}] regionserver.HStore(327): Store=00fd1046c94e7f507c0ee43e6555d034/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:41:00,379 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for 00fd1046c94e7f507c0ee43e6555d034 2024-12-11T22:41:00,380 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/00fd1046c94e7f507c0ee43e6555d034 2024-12-11T22:41:00,381 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for 5a93ec512c97cf46e8942b91f4c8f088 2024-12-11T22:41:00,381 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/00fd1046c94e7f507c0ee43e6555d034 2024-12-11T22:41:00,382 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1733956859502.65f30d8a32c00877e2cf6f22e7f50620. 2024-12-11T22:41:00,382 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,nnn,1733956859502.65f30d8a32c00877e2cf6f22e7f50620. 2024-12-11T22:41:00,382 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for 00fd1046c94e7f507c0ee43e6555d034 2024-12-11T22:41:00,382 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,www,1733956859502.860d604009b38831efaa4f23b2d17596. 2024-12-11T22:41:00,382 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for 00fd1046c94e7f507c0ee43e6555d034 2024-12-11T22:41:00,382 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 860d604009b38831efaa4f23b2d17596, NAME => 'test,www,1733956859502.860d604009b38831efaa4f23b2d17596.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-12-11T22:41:00,383 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 860d604009b38831efaa4f23b2d17596 2024-12-11T22:41:00,383 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,www,1733956859502.860d604009b38831efaa4f23b2d17596.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:41:00,383 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 860d604009b38831efaa4f23b2d17596 2024-12-11T22:41:00,383 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 860d604009b38831efaa4f23b2d17596 2024-12-11T22:41:00,384 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=23 2024-12-11T22:41:00,384 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure daa100e1546a41747fac83159daee190, server=56d7a8405f4a,34703,1733956857011 in 285 msec 2024-12-11T22:41:00,384 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/5a93ec512c97cf46e8942b91f4c8f088/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T22:41:00,386 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened 5a93ec512c97cf46e8942b91f4c8f088; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62061365, jitterRate=-0.07521359622478485}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:41:00,386 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5a93ec512c97cf46e8942b91f4c8f088 2024-12-11T22:41:00,386 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for 5a93ec512c97cf46e8942b91f4c8f088: Running coprocessor pre-open hook at 1733956860359Writing region info on filesystem at 1733956860359Initializing all the Stores at 1733956860363 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733956860363Cleaning up temporary data from old regions at 1733956860377 (+14 ms)Running coprocessor post-open hooks at 1733956860386 (+9 ms)Region opened successfully at 1733956860386 2024-12-11T22:41:00,387 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1733956859502.5a93ec512c97cf46e8942b91f4c8f088., pid=52, masterSystemTime=1733956860167 2024-12-11T22:41:00,388 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for 00fd1046c94e7f507c0ee43e6555d034 2024-12-11T22:41:00,388 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=daa100e1546a41747fac83159daee190, ASSIGN in 565 msec 2024-12-11T22:41:00,388 INFO [StoreOpener-860d604009b38831efaa4f23b2d17596-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 860d604009b38831efaa4f23b2d17596 2024-12-11T22:41:00,391 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=65f30d8a32c00877e2cf6f22e7f50620, regionState=OPEN, openSeqNum=2, regionLocation=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:00,391 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1733956859502.5a93ec512c97cf46e8942b91f4c8f088. 2024-12-11T22:41:00,391 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,,1733956859502.5a93ec512c97cf46e8942b91f4c8f088. 2024-12-11T22:41:00,391 INFO [StoreOpener-860d604009b38831efaa4f23b2d17596-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 860d604009b38831efaa4f23b2d17596 columnFamilyName family 2024-12-11T22:41:00,391 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,ppp,1733956859502.01df3c1c862de5abd063d235e117a115. 2024-12-11T22:41:00,392 DEBUG [StoreOpener-860d604009b38831efaa4f23b2d17596-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:41:00,392 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => 01df3c1c862de5abd063d235e117a115, NAME => 'test,ppp,1733956859502.01df3c1c862de5abd063d235e117a115.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-12-11T22:41:00,392 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 01df3c1c862de5abd063d235e117a115 2024-12-11T22:41:00,392 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,ppp,1733956859502.01df3c1c862de5abd063d235e117a115.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:41:00,392 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for 01df3c1c862de5abd063d235e117a115 2024-12-11T22:41:00,392 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for 01df3c1c862de5abd063d235e117a115 2024-12-11T22:41:00,392 INFO [StoreOpener-860d604009b38831efaa4f23b2d17596-1 {}] regionserver.HStore(327): Store=860d604009b38831efaa4f23b2d17596/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:41:00,393 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 860d604009b38831efaa4f23b2d17596 2024-12-11T22:41:00,394 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/860d604009b38831efaa4f23b2d17596 2024-12-11T22:41:00,394 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/860d604009b38831efaa4f23b2d17596 2024-12-11T22:41:00,394 INFO [StoreOpener-01df3c1c862de5abd063d235e117a115-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 01df3c1c862de5abd063d235e117a115 2024-12-11T22:41:00,395 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 860d604009b38831efaa4f23b2d17596 2024-12-11T22:41:00,395 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 860d604009b38831efaa4f23b2d17596 2024-12-11T22:41:00,395 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/00fd1046c94e7f507c0ee43e6555d034/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T22:41:00,396 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 65f30d8a32c00877e2cf6f22e7f50620, server=56d7a8405f4a,40421,1733956857131 because future has completed 2024-12-11T22:41:00,397 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened 00fd1046c94e7f507c0ee43e6555d034; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64566434, jitterRate=-0.0378851592540741}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:41:00,397 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 00fd1046c94e7f507c0ee43e6555d034 2024-12-11T22:41:00,398 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for 00fd1046c94e7f507c0ee43e6555d034: Running coprocessor pre-open hook at 1733956860361Writing region info on filesystem at 1733956860361Initializing all the Stores at 1733956860364 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733956860364Cleaning up temporary data from old regions at 1733956860382 (+18 ms)Running coprocessor post-open hooks at 1733956860397 (+15 ms)Region opened successfully at 1733956860397 2024-12-11T22:41:00,398 INFO [StoreOpener-01df3c1c862de5abd063d235e117a115-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 01df3c1c862de5abd063d235e117a115 columnFamilyName family 2024-12-11T22:41:00,398 DEBUG [StoreOpener-01df3c1c862de5abd063d235e117a115-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:41:00,399 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1733956859502.00fd1046c94e7f507c0ee43e6555d034., pid=56, masterSystemTime=1733956860237 2024-12-11T22:41:00,399 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 860d604009b38831efaa4f23b2d17596 2024-12-11T22:41:00,399 INFO [StoreOpener-01df3c1c862de5abd063d235e117a115-1 {}] regionserver.HStore(327): Store=01df3c1c862de5abd063d235e117a115/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:41:00,399 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for 01df3c1c862de5abd063d235e117a115 2024-12-11T22:41:00,399 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5a93ec512c97cf46e8942b91f4c8f088, regionState=OPEN, openSeqNum=2, regionLocation=56d7a8405f4a,36971,1733956857186 2024-12-11T22:41:00,400 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/01df3c1c862de5abd063d235e117a115 2024-12-11T22:41:00,402 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/01df3c1c862de5abd063d235e117a115 2024-12-11T22:41:00,403 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for 01df3c1c862de5abd063d235e117a115 2024-12-11T22:41:00,403 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for 01df3c1c862de5abd063d235e117a115 2024-12-11T22:41:00,404 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1733956859502.00fd1046c94e7f507c0ee43e6555d034. 2024-12-11T22:41:00,404 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,iii,1733956859502.00fd1046c94e7f507c0ee43e6555d034. 2024-12-11T22:41:00,404 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,kkk,1733956859502.6561ead2eaa9f60da54d4ccff5474a28. 2024-12-11T22:41:00,404 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => 6561ead2eaa9f60da54d4ccff5474a28, NAME => 'test,kkk,1733956859502.6561ead2eaa9f60da54d4ccff5474a28.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-12-11T22:41:00,404 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,36971,1733956857186 because future has completed 2024-12-11T22:41:00,404 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6561ead2eaa9f60da54d4ccff5474a28 2024-12-11T22:41:00,405 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,kkk,1733956859502.6561ead2eaa9f60da54d4ccff5474a28.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:41:00,405 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for 6561ead2eaa9f60da54d4ccff5474a28 2024-12-11T22:41:00,405 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for 6561ead2eaa9f60da54d4ccff5474a28 2024-12-11T22:41:00,406 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for 01df3c1c862de5abd063d235e117a115 2024-12-11T22:41:00,408 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/860d604009b38831efaa4f23b2d17596/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T22:41:00,408 INFO [StoreOpener-6561ead2eaa9f60da54d4ccff5474a28-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6561ead2eaa9f60da54d4ccff5474a28 2024-12-11T22:41:00,409 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 860d604009b38831efaa4f23b2d17596; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71548032, jitterRate=0.06614875793457031}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:41:00,410 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 860d604009b38831efaa4f23b2d17596 2024-12-11T22:41:00,410 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 860d604009b38831efaa4f23b2d17596: Running coprocessor pre-open hook at 1733956860383Writing region info on filesystem at 1733956860383Initializing all the Stores at 1733956860387 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733956860387Cleaning up temporary data from old regions at 1733956860395 (+8 ms)Running coprocessor post-open hooks at 1733956860410 (+15 ms)Region opened successfully at 1733956860410 2024-12-11T22:41:00,411 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/01df3c1c862de5abd063d235e117a115/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T22:41:00,411 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=18 2024-12-11T22:41:00,411 INFO [StoreOpener-6561ead2eaa9f60da54d4ccff5474a28-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6561ead2eaa9f60da54d4ccff5474a28 columnFamilyName family 2024-12-11T22:41:00,411 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure 65f30d8a32c00877e2cf6f22e7f50620, server=56d7a8405f4a,40421,1733956857131 in 373 msec 2024-12-11T22:41:00,411 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened 01df3c1c862de5abd063d235e117a115; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65053436, jitterRate=-0.0306282639503479}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:41:00,412 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 01df3c1c862de5abd063d235e117a115 2024-12-11T22:41:00,412 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for 01df3c1c862de5abd063d235e117a115: Running coprocessor pre-open hook at 1733956860393Writing region info on filesystem at 1733956860393Initializing all the Stores at 1733956860394 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733956860394Cleaning up temporary data from old regions at 1733956860403 (+9 ms)Running coprocessor post-open hooks at 1733956860412 (+9 ms)Region opened successfully at 1733956860412 2024-12-11T22:41:00,411 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1733956859502.860d604009b38831efaa4f23b2d17596., pid=40, masterSystemTime=1733956860148 2024-12-11T22:41:00,411 DEBUG [StoreOpener-6561ead2eaa9f60da54d4ccff5474a28-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:41:00,413 INFO [StoreOpener-6561ead2eaa9f60da54d4ccff5474a28-1 {}] regionserver.HStore(327): Store=6561ead2eaa9f60da54d4ccff5474a28/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:41:00,413 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1733956859502.01df3c1c862de5abd063d235e117a115., pid=45, masterSystemTime=1733956860167 2024-12-11T22:41:00,413 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for 6561ead2eaa9f60da54d4ccff5474a28 2024-12-11T22:41:00,415 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/6561ead2eaa9f60da54d4ccff5474a28 2024-12-11T22:41:00,416 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/6561ead2eaa9f60da54d4ccff5474a28 2024-12-11T22:41:00,416 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for 6561ead2eaa9f60da54d4ccff5474a28 2024-12-11T22:41:00,416 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for 6561ead2eaa9f60da54d4ccff5474a28 2024-12-11T22:41:00,417 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1733956859502.860d604009b38831efaa4f23b2d17596. 2024-12-11T22:41:00,417 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,www,1733956859502.860d604009b38831efaa4f23b2d17596. 2024-12-11T22:41:00,418 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=65f30d8a32c00877e2cf6f22e7f50620, ASSIGN in 593 msec 2024-12-11T22:41:00,418 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1733956859502.01df3c1c862de5abd063d235e117a115. 2024-12-11T22:41:00,418 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,ppp,1733956859502.01df3c1c862de5abd063d235e117a115. 2024-12-11T22:41:00,419 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,fff,1733956859502.419f7368f61981f0a715c20304b74e0f. 2024-12-11T22:41:00,419 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => 419f7368f61981f0a715c20304b74e0f, NAME => 'test,fff,1733956859502.419f7368f61981f0a715c20304b74e0f.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-12-11T22:41:00,419 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 419f7368f61981f0a715c20304b74e0f 2024-12-11T22:41:00,419 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,fff,1733956859502.419f7368f61981f0a715c20304b74e0f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:41:00,419 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for 6561ead2eaa9f60da54d4ccff5474a28 2024-12-11T22:41:00,419 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for 419f7368f61981f0a715c20304b74e0f 2024-12-11T22:41:00,419 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for 419f7368f61981f0a715c20304b74e0f 2024-12-11T22:41:00,421 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=01df3c1c862de5abd063d235e117a115, regionState=OPEN, openSeqNum=2, regionLocation=56d7a8405f4a,36971,1733956857186 2024-12-11T22:41:00,421 INFO [StoreOpener-419f7368f61981f0a715c20304b74e0f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 419f7368f61981f0a715c20304b74e0f 2024-12-11T22:41:00,425 INFO [StoreOpener-419f7368f61981f0a715c20304b74e0f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 419f7368f61981f0a715c20304b74e0f columnFamilyName family 2024-12-11T22:41:00,425 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=860d604009b38831efaa4f23b2d17596, regionState=OPEN, openSeqNum=2, regionLocation=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:00,425 DEBUG [StoreOpener-419f7368f61981f0a715c20304b74e0f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:41:00,425 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 01df3c1c862de5abd063d235e117a115, server=56d7a8405f4a,36971,1733956857186 because future has completed 2024-12-11T22:41:00,426 INFO [StoreOpener-419f7368f61981f0a715c20304b74e0f-1 {}] regionserver.HStore(327): Store=419f7368f61981f0a715c20304b74e0f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:41:00,426 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for 419f7368f61981f0a715c20304b74e0f 2024-12-11T22:41:00,428 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/6561ead2eaa9f60da54d4ccff5474a28/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T22:41:00,428 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/419f7368f61981f0a715c20304b74e0f 2024-12-11T22:41:00,429 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened 6561ead2eaa9f60da54d4ccff5474a28; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74026732, jitterRate=0.1030842661857605}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:41:00,429 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/419f7368f61981f0a715c20304b74e0f 2024-12-11T22:41:00,429 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6561ead2eaa9f60da54d4ccff5474a28 2024-12-11T22:41:00,429 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for 6561ead2eaa9f60da54d4ccff5474a28: Running coprocessor pre-open hook at 1733956860405Writing region info on filesystem at 1733956860405Initializing all the Stores at 1733956860408 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733956860408Cleaning up temporary data from old regions at 1733956860416 (+8 ms)Running coprocessor post-open hooks at 1733956860429 (+13 ms)Region opened successfully at 1733956860429 2024-12-11T22:41:00,430 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for 419f7368f61981f0a715c20304b74e0f 2024-12-11T22:41:00,431 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for 419f7368f61981f0a715c20304b74e0f 2024-12-11T22:41:00,431 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34843 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=56d7a8405f4a,40421,1733956857131, table=test, region=445df7fd625f94e0eb0712fcd777ebda. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-11T22:41:00,431 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34843 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=56d7a8405f4a,40421,1733956857131, table=test, region=860d604009b38831efaa4f23b2d17596. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-11T22:41:00,432 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 860d604009b38831efaa4f23b2d17596, server=56d7a8405f4a,40421,1733956857131 because future has completed 2024-12-11T22:41:00,433 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for 419f7368f61981f0a715c20304b74e0f 2024-12-11T22:41:00,434 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1733956859502.6561ead2eaa9f60da54d4ccff5474a28., pid=53, masterSystemTime=1733956860237 2024-12-11T22:41:00,438 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1733956859502.6561ead2eaa9f60da54d4ccff5474a28. 2024-12-11T22:41:00,438 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,kkk,1733956859502.6561ead2eaa9f60da54d4ccff5474a28. 2024-12-11T22:41:00,438 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,ccc,1733956859502.6801ac4c1cb1416e045d462c592d2dc5. 2024-12-11T22:41:00,439 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => 6801ac4c1cb1416e045d462c592d2dc5, NAME => 'test,ccc,1733956859502.6801ac4c1cb1416e045d462c592d2dc5.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-12-11T22:41:00,439 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6801ac4c1cb1416e045d462c592d2dc5 2024-12-11T22:41:00,439 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/419f7368f61981f0a715c20304b74e0f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T22:41:00,439 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,ccc,1733956859502.6801ac4c1cb1416e045d462c592d2dc5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:41:00,440 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for 6801ac4c1cb1416e045d462c592d2dc5 2024-12-11T22:41:00,440 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for 6801ac4c1cb1416e045d462c592d2dc5 2024-12-11T22:41:00,440 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened 419f7368f61981f0a715c20304b74e0f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69863193, jitterRate=0.04104270040988922}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:41:00,440 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 419f7368f61981f0a715c20304b74e0f 2024-12-11T22:41:00,440 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for 419f7368f61981f0a715c20304b74e0f: Running coprocessor pre-open hook at 1733956860419Writing region info on filesystem at 1733956860419Initializing all the Stores at 1733956860421 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733956860421Cleaning up temporary data from old regions at 1733956860431 (+10 ms)Running coprocessor post-open hooks at 1733956860440 (+9 ms)Region opened successfully at 1733956860440 2024-12-11T22:41:00,441 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=20 2024-12-11T22:41:00,441 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 01df3c1c862de5abd063d235e117a115, server=56d7a8405f4a,36971,1733956857186 in 367 msec 2024-12-11T22:41:00,441 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34843 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=56d7a8405f4a,36971,1733956857186, table=test, region=5a93ec512c97cf46e8942b91f4c8f088. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-11T22:41:00,442 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1733956859502.419f7368f61981f0a715c20304b74e0f., pid=47, masterSystemTime=1733956860167 2024-12-11T22:41:00,445 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN in 622 msec 2024-12-11T22:41:00,445 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1733956859502.419f7368f61981f0a715c20304b74e0f. 2024-12-11T22:41:00,445 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,fff,1733956859502.419f7368f61981f0a715c20304b74e0f. 2024-12-11T22:41:00,446 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=419f7368f61981f0a715c20304b74e0f, regionState=OPEN, openSeqNum=2, regionLocation=56d7a8405f4a,36971,1733956857186 2024-12-11T22:41:00,447 INFO [StoreOpener-6801ac4c1cb1416e045d462c592d2dc5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6801ac4c1cb1416e045d462c592d2dc5 2024-12-11T22:41:00,449 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 419f7368f61981f0a715c20304b74e0f, server=56d7a8405f4a,36971,1733956857186 because future has completed 2024-12-11T22:41:00,450 INFO [StoreOpener-6801ac4c1cb1416e045d462c592d2dc5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6801ac4c1cb1416e045d462c592d2dc5 columnFamilyName family 2024-12-11T22:41:00,450 DEBUG [StoreOpener-6801ac4c1cb1416e045d462c592d2dc5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:41:00,451 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=6561ead2eaa9f60da54d4ccff5474a28, regionState=OPEN, openSeqNum=2, regionLocation=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:00,451 INFO [StoreOpener-6801ac4c1cb1416e045d462c592d2dc5-1 {}] regionserver.HStore(327): Store=6801ac4c1cb1416e045d462c592d2dc5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:41:00,451 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for 6801ac4c1cb1416e045d462c592d2dc5 2024-12-11T22:41:00,452 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/6801ac4c1cb1416e045d462c592d2dc5 2024-12-11T22:41:00,453 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/6801ac4c1cb1416e045d462c592d2dc5 2024-12-11T22:41:00,454 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for 6801ac4c1cb1416e045d462c592d2dc5 2024-12-11T22:41:00,454 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for 6801ac4c1cb1416e045d462c592d2dc5 2024-12-11T22:41:00,455 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6561ead2eaa9f60da54d4ccff5474a28, server=56d7a8405f4a,34703,1733956857011 because future has completed 2024-12-11T22:41:00,457 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for 6801ac4c1cb1416e045d462c592d2dc5 2024-12-11T22:41:00,460 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=10 2024-12-11T22:41:00,460 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 419f7368f61981f0a715c20304b74e0f, server=56d7a8405f4a,36971,1733956857186 in 376 msec 2024-12-11T22:41:00,461 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/6801ac4c1cb1416e045d462c592d2dc5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T22:41:00,462 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened 6801ac4c1cb1416e045d462c592d2dc5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69003973, jitterRate=0.028239324688911438}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:41:00,462 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6801ac4c1cb1416e045d462c592d2dc5 2024-12-11T22:41:00,462 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for 6801ac4c1cb1416e045d462c592d2dc5: Running coprocessor pre-open hook at 1733956860440Writing region info on filesystem at 1733956860440Initializing all the Stores at 1733956860443 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733956860443Cleaning up temporary data from old regions at 1733956860454 (+11 ms)Running coprocessor post-open hooks at 1733956860462 (+8 ms)Region opened successfully at 1733956860462 2024-12-11T22:41:00,463 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1733956859502.6801ac4c1cb1416e045d462c592d2dc5., pid=54, masterSystemTime=1733956860237 2024-12-11T22:41:00,464 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN in 641 msec 2024-12-11T22:41:00,466 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1733956859502.6801ac4c1cb1416e045d462c592d2dc5. 2024-12-11T22:41:00,466 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,ccc,1733956859502.6801ac4c1cb1416e045d462c592d2dc5. 2024-12-11T22:41:00,467 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,eee,1733956859502.e9e94fba4d03035b5feaab607eb13853. 2024-12-11T22:41:00,467 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => e9e94fba4d03035b5feaab607eb13853, NAME => 'test,eee,1733956859502.e9e94fba4d03035b5feaab607eb13853.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-12-11T22:41:00,467 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e9e94fba4d03035b5feaab607eb13853 2024-12-11T22:41:00,468 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,eee,1733956859502.e9e94fba4d03035b5feaab607eb13853.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:41:00,468 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for e9e94fba4d03035b5feaab607eb13853 2024-12-11T22:41:00,468 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for e9e94fba4d03035b5feaab607eb13853 2024-12-11T22:41:00,469 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=15 2024-12-11T22:41:00,469 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure 6561ead2eaa9f60da54d4ccff5474a28, server=56d7a8405f4a,34703,1733956857011 in 357 msec 2024-12-11T22:41:00,470 INFO [StoreOpener-e9e94fba4d03035b5feaab607eb13853-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e9e94fba4d03035b5feaab607eb13853 2024-12-11T22:41:00,472 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6561ead2eaa9f60da54d4ccff5474a28, ASSIGN in 651 msec 2024-12-11T22:41:00,473 INFO [StoreOpener-e9e94fba4d03035b5feaab607eb13853-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e9e94fba4d03035b5feaab607eb13853 columnFamilyName family 2024-12-11T22:41:00,473 DEBUG [StoreOpener-e9e94fba4d03035b5feaab607eb13853-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:41:00,474 INFO [StoreOpener-e9e94fba4d03035b5feaab607eb13853-1 {}] regionserver.HStore(327): Store=e9e94fba4d03035b5feaab607eb13853/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:41:00,474 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=6801ac4c1cb1416e045d462c592d2dc5, regionState=OPEN, openSeqNum=2, regionLocation=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:00,474 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for e9e94fba4d03035b5feaab607eb13853 2024-12-11T22:41:00,475 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/e9e94fba4d03035b5feaab607eb13853 2024-12-11T22:41:00,476 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/e9e94fba4d03035b5feaab607eb13853 2024-12-11T22:41:00,477 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6801ac4c1cb1416e045d462c592d2dc5, server=56d7a8405f4a,34703,1733956857011 because future has completed 2024-12-11T22:41:00,478 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for e9e94fba4d03035b5feaab607eb13853 2024-12-11T22:41:00,478 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for e9e94fba4d03035b5feaab607eb13853 2024-12-11T22:41:00,481 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for e9e94fba4d03035b5feaab607eb13853 2024-12-11T22:41:00,481 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=27 2024-12-11T22:41:00,481 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 860d604009b38831efaa4f23b2d17596, server=56d7a8405f4a,40421,1733956857131 in 438 msec 2024-12-11T22:41:00,485 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/e9e94fba4d03035b5feaab607eb13853/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T22:41:00,486 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=860d604009b38831efaa4f23b2d17596, ASSIGN in 662 msec 2024-12-11T22:41:00,486 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened e9e94fba4d03035b5feaab607eb13853; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73702643, jitterRate=0.0982549637556076}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:41:00,486 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e9e94fba4d03035b5feaab607eb13853 2024-12-11T22:41:00,486 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for e9e94fba4d03035b5feaab607eb13853: Running coprocessor pre-open hook at 1733956860468Writing region info on filesystem at 1733956860468Initializing all the Stores at 1733956860469 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733956860469Cleaning up temporary data from old regions at 1733956860478 (+9 ms)Running coprocessor post-open hooks at 1733956860486 (+8 ms)Region opened successfully at 1733956860486 2024-12-11T22:41:00,488 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1733956859502.e9e94fba4d03035b5feaab607eb13853., pid=55, masterSystemTime=1733956860237 2024-12-11T22:41:00,490 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1733956859502.e9e94fba4d03035b5feaab607eb13853. 2024-12-11T22:41:00,490 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,eee,1733956859502.e9e94fba4d03035b5feaab607eb13853. 2024-12-11T22:41:00,491 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,yyy,1733956859502.f19f540286846a30911c41a83b97457e. 2024-12-11T22:41:00,491 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=7 2024-12-11T22:41:00,491 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => f19f540286846a30911c41a83b97457e, NAME => 'test,yyy,1733956859502.f19f540286846a30911c41a83b97457e.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-12-11T22:41:00,491 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 6801ac4c1cb1416e045d462c592d2dc5, server=56d7a8405f4a,34703,1733956857011 in 371 msec 2024-12-11T22:41:00,491 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f19f540286846a30911c41a83b97457e 2024-12-11T22:41:00,491 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,yyy,1733956859502.f19f540286846a30911c41a83b97457e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:41:00,491 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for f19f540286846a30911c41a83b97457e 2024-12-11T22:41:00,491 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for f19f540286846a30911c41a83b97457e 2024-12-11T22:41:00,493 INFO [StoreOpener-f19f540286846a30911c41a83b97457e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f19f540286846a30911c41a83b97457e 2024-12-11T22:41:00,494 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6801ac4c1cb1416e045d462c592d2dc5, ASSIGN in 672 msec 2024-12-11T22:41:00,495 INFO [StoreOpener-f19f540286846a30911c41a83b97457e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f19f540286846a30911c41a83b97457e columnFamilyName family 2024-12-11T22:41:00,495 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=e9e94fba4d03035b5feaab607eb13853, regionState=OPEN, openSeqNum=2, regionLocation=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:00,496 DEBUG [StoreOpener-f19f540286846a30911c41a83b97457e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:41:00,496 INFO [StoreOpener-f19f540286846a30911c41a83b97457e-1 {}] regionserver.HStore(327): Store=f19f540286846a30911c41a83b97457e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:41:00,497 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for f19f540286846a30911c41a83b97457e 2024-12-11T22:41:00,499 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/f19f540286846a30911c41a83b97457e 2024-12-11T22:41:00,500 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure e9e94fba4d03035b5feaab607eb13853, server=56d7a8405f4a,34703,1733956857011 because future has completed 2024-12-11T22:41:00,500 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/f19f540286846a30911c41a83b97457e 2024-12-11T22:41:00,501 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for f19f540286846a30911c41a83b97457e 2024-12-11T22:41:00,501 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for f19f540286846a30911c41a83b97457e 2024-12-11T22:41:00,504 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=5 2024-12-11T22:41:00,504 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,36971,1733956857186 in 398 msec 2024-12-11T22:41:00,504 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for f19f540286846a30911c41a83b97457e 2024-12-11T22:41:00,507 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN in 685 msec 2024-12-11T22:41:00,509 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/f19f540286846a30911c41a83b97457e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T22:41:00,510 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened f19f540286846a30911c41a83b97457e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61547101, jitterRate=-0.08287672698497772}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:41:00,510 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f19f540286846a30911c41a83b97457e 2024-12-11T22:41:00,510 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for f19f540286846a30911c41a83b97457e: Running coprocessor pre-open hook at 1733956860491Writing region info on filesystem at 1733956860491Initializing all the Stores at 1733956860493 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733956860493Cleaning up temporary data from old regions at 1733956860501 (+8 ms)Running coprocessor post-open hooks at 1733956860510 (+9 ms)Region opened successfully at 1733956860510 2024-12-11T22:41:00,511 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1733956859502.f19f540286846a30911c41a83b97457e., pid=51, masterSystemTime=1733956860237 2024-12-11T22:41:00,513 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=9 2024-12-11T22:41:00,513 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure e9e94fba4d03035b5feaab607eb13853, server=56d7a8405f4a,34703,1733956857011 in 388 msec 2024-12-11T22:41:00,515 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1733956859502.f19f540286846a30911c41a83b97457e. 2024-12-11T22:41:00,515 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,yyy,1733956859502.f19f540286846a30911c41a83b97457e. 2024-12-11T22:41:00,516 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e9e94fba4d03035b5feaab607eb13853, ASSIGN in 694 msec 2024-12-11T22:41:00,517 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=f19f540286846a30911c41a83b97457e, regionState=OPEN, openSeqNum=2, regionLocation=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:00,520 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=00fd1046c94e7f507c0ee43e6555d034, regionState=OPEN, openSeqNum=2, regionLocation=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:00,520 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure f19f540286846a30911c41a83b97457e, server=56d7a8405f4a,34703,1733956857011 because future has completed 2024-12-11T22:41:00,523 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 00fd1046c94e7f507c0ee43e6555d034, server=56d7a8405f4a,34703,1733956857011 because future has completed 2024-12-11T22:41:00,527 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=29 2024-12-11T22:41:00,527 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure f19f540286846a30911c41a83b97457e, server=56d7a8405f4a,34703,1733956857011 in 424 msec 2024-12-11T22:41:00,530 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f19f540286846a30911c41a83b97457e, ASSIGN in 708 msec 2024-12-11T22:41:00,534 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=13 2024-12-11T22:41:00,535 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure 00fd1046c94e7f507c0ee43e6555d034, server=56d7a8405f4a,34703,1733956857011 in 406 msec 2024-12-11T22:41:00,536 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34843 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=56d7a8405f4a,40421,1733956857131, table=test, region=445df7fd625f94e0eb0712fcd777ebda. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-11T22:41:00,538 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=00fd1046c94e7f507c0ee43e6555d034, ASSIGN in 716 msec 2024-12-11T22:41:00,539 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=2f551d316d038ca1e1403bd05f890572, regionState=OPEN, openSeqNum=2, regionLocation=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:00,542 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2f551d316d038ca1e1403bd05f890572, server=56d7a8405f4a,40421,1733956857131 because future has completed 2024-12-11T22:41:00,543 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=b1a38f11b48b53983535b863fd4392ff, regionState=OPEN, openSeqNum=2, regionLocation=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:00,548 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure b1a38f11b48b53983535b863fd4392ff, server=56d7a8405f4a,34703,1733956857011 because future has completed 2024-12-11T22:41:00,551 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=21 2024-12-11T22:41:00,551 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 2f551d316d038ca1e1403bd05f890572, server=56d7a8405f4a,40421,1733956857131 in 546 msec 2024-12-11T22:41:00,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=2f551d316d038ca1e1403bd05f890572, ASSIGN in 733 msec 2024-12-11T22:41:00,559 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=19 2024-12-11T22:41:00,559 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure b1a38f11b48b53983535b863fd4392ff, server=56d7a8405f4a,34703,1733956857011 in 466 msec 2024-12-11T22:41:00,562 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b1a38f11b48b53983535b863fd4392ff, ASSIGN in 740 msec 2024-12-11T22:41:00,566 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=16 2024-12-11T22:41:00,566 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 445df7fd625f94e0eb0712fcd777ebda, server=56d7a8405f4a,40421,1733956857131 in 536 msec 2024-12-11T22:41:00,569 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=445df7fd625f94e0eb0712fcd777ebda, ASSIGN in 747 msec 2024-12-11T22:41:00,570 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=a0e02a7a4f8d57169d1f435eabc00839, regionState=OPEN, openSeqNum=2, regionLocation=56d7a8405f4a,36971,1733956857186 2024-12-11T22:41:00,573 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0e02a7a4f8d57169d1f435eabc00839, server=56d7a8405f4a,36971,1733956857186 because future has completed 2024-12-11T22:41:00,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=28 2024-12-11T22:41:00,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure a0e02a7a4f8d57169d1f435eabc00839, server=56d7a8405f4a,36971,1733956857186 in 560 msec 2024-12-11T22:41:00,582 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=4 2024-12-11T22:41:00,582 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN in 759 msec 2024-12-11T22:41:00,583 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-11T22:41:00,584 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733956860584"}]},"ts":"1733956860584"} 2024-12-11T22:41:00,586 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-12-11T22:41:00,589 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-12-11T22:41:00,595 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 1.0800 sec 2024-12-11T22:41:00,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-11T22:41:00,666 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-12-11T22:41:00,666 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-12-11T22:41:00,668 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T22:41:00,687 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-12-11T22:41:00,688 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T22:41:00,689 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-12-11T22:41:00,690 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T22:41:00,695 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-12-11T22:41:00,716 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-12-11T22:41:00,730 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T22:41:00,732 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T22:41:00,735 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T22:41:00,737 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T22:41:00,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6801ac4c1cb1416e045d462c592d2dc5 changed from -1.0 to 0.0, refreshing cache 2024-12-11T22:41:00,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e9e94fba4d03035b5feaab607eb13853 changed from -1.0 to 0.0, refreshing cache 2024-12-11T22:41:00,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 00fd1046c94e7f507c0ee43e6555d034 changed from -1.0 to 0.0, refreshing cache 2024-12-11T22:41:00,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6561ead2eaa9f60da54d4ccff5474a28 changed from -1.0 to 0.0, refreshing cache 2024-12-11T22:41:00,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b1a38f11b48b53983535b863fd4392ff changed from -1.0 to 0.0, refreshing cache 2024-12-11T22:41:00,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region daa100e1546a41747fac83159daee190 changed from -1.0 to 0.0, refreshing cache 2024-12-11T22:41:00,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f7dbe11f45ae33a883424947e613c7c6 changed from -1.0 to 0.0, refreshing cache 2024-12-11T22:41:00,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f19f540286846a30911c41a83b97457e changed from -1.0 to 0.0, refreshing cache 2024-12-11T22:41:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5a93ec512c97cf46e8942b91f4c8f088 changed from -1.0 to 0.0, refreshing cache 2024-12-11T22:41:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 419f7368f61981f0a715c20304b74e0f changed from -1.0 to 0.0, refreshing cache 2024-12-11T22:41:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region de3803c768bc063bdc71e495af58eb48 changed from -1.0 to 0.0, refreshing cache 2024-12-11T22:41:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ad4bb3db7f3f05f24b6115fc887d6ab5 changed from -1.0 to 0.0, refreshing cache 2024-12-11T22:41:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f666d29f964372d8719d169fcecef641 changed from -1.0 to 0.0, refreshing cache 2024-12-11T22:41:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 01df3c1c862de5abd063d235e117a115 changed from -1.0 to 0.0, refreshing cache 2024-12-11T22:41:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f219e06826a4a3ede110adacdf28f6f4 changed from -1.0 to 0.0, refreshing cache 2024-12-11T22:41:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b30c1b01a8241f23347f03f93df7f090 changed from -1.0 to 0.0, refreshing cache 2024-12-11T22:41:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a0e02a7a4f8d57169d1f435eabc00839 changed from -1.0 to 0.0, refreshing cache 2024-12-11T22:41:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 66aac9e74e8263b0b8914fdc86eec66c changed from -1.0 to 0.0, refreshing cache 2024-12-11T22:41:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 9f169feb3190772cec56c0ae51d57c26 changed from -1.0 to 0.0, refreshing cache 2024-12-11T22:41:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b31ee44b6609c355adc37986b13d239c changed from -1.0 to 0.0, refreshing cache 2024-12-11T22:41:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 445df7fd625f94e0eb0712fcd777ebda changed from -1.0 to 0.0, refreshing cache 2024-12-11T22:41:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 65f30d8a32c00877e2cf6f22e7f50620 changed from -1.0 to 0.0, refreshing cache 2024-12-11T22:41:00,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2f551d316d038ca1e1403bd05f890572 changed from -1.0 to 0.0, refreshing cache 2024-12-11T22:41:00,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region da518288db554a985096494f24adc59f changed from -1.0 to 0.0, refreshing cache 2024-12-11T22:41:00,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 860d604009b38831efaa4f23b2d17596 changed from -1.0 to 0.0, refreshing cache 2024-12-11T22:41:00,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ff9e1faa3192d62c84407efca19e0a89 changed from -1.0 to 0.0, refreshing cache 2024-12-11T22:41:00,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-12-11T22:41:00,791 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-12-11T22:41:00,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-12-11T22:41:00,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] balancer.BalancerClusterState(204): Hosts are {56d7a8405f4a=0} racks are {/default-rack=0} 2024-12-11T22:41:00,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] balancer.BalancerClusterState(303): server 0 has 8 regions 2024-12-11T22:41:00,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] balancer.BalancerClusterState(303): server 1 has 10 regions 2024-12-11T22:41:00,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] balancer.BalancerClusterState(303): server 2 has 9 regions 2024-12-11T22:41:00,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-11T22:41:00,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-11T22:41:00,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-11T22:41:00,792 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-11T22:41:00,793 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-11T22:41:00,793 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-11T22:41:00,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-12-11T22:41:00,797 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] balancer.StochasticLoadBalancer(403): Cluster wide - Calculating plan. may take up to 30000ms to complete. 2024-12-11T22:41:00,799 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] balancer.StochasticLoadBalancer(515): Start StochasticLoadBalancer.balancer, initial weighted average imbalance=0.06931633435746164, functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.06415002990995843, need balance); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); computedMaxSteps=64800 2024-12-11T22:41:01,206 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] balancer.StochasticLoadBalancer(562): Finished computing new moving plan. Computation took 413 ms to try 64800 different iterations. Found a solution that moves 1 regions; Going from a computed imbalance of 0.06931633435746164 to a new imbalance of 0.01690158032826896. funtionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.037037037037037035, need balance); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-12-11T22:41:01,209 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] master.HMaster(2167): Balancer plans size is 1, the balance interval is 300000 ms, and the max number regions in transition is 27 2024-12-11T22:41:01,209 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] master.HMaster(2172): balance hri=f666d29f964372d8719d169fcecef641, source=56d7a8405f4a,36971,1733956857186, destination=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:01,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=f666d29f964372d8719d169fcecef641, REOPEN/MOVE 2024-12-11T22:41:01,211 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=f666d29f964372d8719d169fcecef641, REOPEN/MOVE 2024-12-11T22:41:01,213 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=f666d29f964372d8719d169fcecef641, regionState=CLOSING, regionLocation=56d7a8405f4a,36971,1733956857186 2024-12-11T22:41:01,216 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test, region=f666d29f964372d8719d169fcecef641, REOPEN/MOVE because future has completed 2024-12-11T22:41:01,217 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-11T22:41:01,217 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure f666d29f964372d8719d169fcecef641, server=56d7a8405f4a,36971,1733956857186}] 2024-12-11T22:41:01,375 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(122): Close f666d29f964372d8719d169fcecef641 2024-12-11T22:41:01,376 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-11T22:41:01,376 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1722): Closing f666d29f964372d8719d169fcecef641, disabling compactions & flushes 2024-12-11T22:41:01,376 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1755): Closing region test,mmm,1733956859502.f666d29f964372d8719d169fcecef641. 2024-12-11T22:41:01,376 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1733956859502.f666d29f964372d8719d169fcecef641. 2024-12-11T22:41:01,376 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1733956859502.f666d29f964372d8719d169fcecef641. after waiting 0 ms 2024-12-11T22:41:01,376 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1733956859502.f666d29f964372d8719d169fcecef641. 2024-12-11T22:41:01,381 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/f666d29f964372d8719d169fcecef641/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-11T22:41:01,383 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1973): Closed test,mmm,1733956859502.f666d29f964372d8719d169fcecef641. 2024-12-11T22:41:01,383 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1676): Region close journal for f666d29f964372d8719d169fcecef641: Waiting for close lock at 1733956861376Running coprocessor pre-close hooks at 1733956861376Disabling compacts and flushes for region at 1733956861376Disabling writes for close at 1733956861376Writing region close event to WAL at 1733956861377 (+1 ms)Running coprocessor post-close hooks at 1733956861382 (+5 ms)Closed at 1733956861383 (+1 ms) 2024-12-11T22:41:01,384 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegionServer(3302): Adding f666d29f964372d8719d169fcecef641 move to 56d7a8405f4a,34703,1733956857011 record at close sequenceid=2 2024-12-11T22:41:01,386 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(157): Closed f666d29f964372d8719d169fcecef641 2024-12-11T22:41:01,387 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=f666d29f964372d8719d169fcecef641, regionState=CLOSED 2024-12-11T22:41:01,389 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure f666d29f964372d8719d169fcecef641, server=56d7a8405f4a,36971,1733956857186 because future has completed 2024-12-11T22:41:01,393 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-12-11T22:41:01,393 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; CloseRegionProcedure f666d29f964372d8719d169fcecef641, server=56d7a8405f4a,36971,1733956857186 in 173 msec 2024-12-11T22:41:01,394 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f666d29f964372d8719d169fcecef641, REOPEN/MOVE; state=CLOSED, location=56d7a8405f4a,34703,1733956857011; forceNewPlan=false, retain=false 2024-12-11T22:41:01,545 INFO [56d7a8405f4a:34843 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-11T22:41:01,546 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=f666d29f964372d8719d169fcecef641, regionState=OPENING, regionLocation=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:01,551 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f666d29f964372d8719d169fcecef641, REOPEN/MOVE because future has completed 2024-12-11T22:41:01,552 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure f666d29f964372d8719d169fcecef641, server=56d7a8405f4a,34703,1733956857011}] 2024-12-11T22:41:01,714 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open test,mmm,1733956859502.f666d29f964372d8719d169fcecef641. 2024-12-11T22:41:01,714 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => f666d29f964372d8719d169fcecef641, NAME => 'test,mmm,1733956859502.f666d29f964372d8719d169fcecef641.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-12-11T22:41:01,715 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f666d29f964372d8719d169fcecef641 2024-12-11T22:41:01,715 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated test,mmm,1733956859502.f666d29f964372d8719d169fcecef641.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:41:01,715 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for f666d29f964372d8719d169fcecef641 2024-12-11T22:41:01,716 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for f666d29f964372d8719d169fcecef641 2024-12-11T22:41:01,718 INFO [StoreOpener-f666d29f964372d8719d169fcecef641-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f666d29f964372d8719d169fcecef641 2024-12-11T22:41:01,720 INFO [StoreOpener-f666d29f964372d8719d169fcecef641-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f666d29f964372d8719d169fcecef641 columnFamilyName family 2024-12-11T22:41:01,720 DEBUG [StoreOpener-f666d29f964372d8719d169fcecef641-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:41:01,721 INFO [StoreOpener-f666d29f964372d8719d169fcecef641-1 {}] regionserver.HStore(327): Store=f666d29f964372d8719d169fcecef641/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:41:01,721 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for f666d29f964372d8719d169fcecef641 2024-12-11T22:41:01,722 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/f666d29f964372d8719d169fcecef641 2024-12-11T22:41:01,723 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/f666d29f964372d8719d169fcecef641 2024-12-11T22:41:01,724 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for f666d29f964372d8719d169fcecef641 2024-12-11T22:41:01,724 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for f666d29f964372d8719d169fcecef641 2024-12-11T22:41:01,726 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for f666d29f964372d8719d169fcecef641 2024-12-11T22:41:01,727 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened f666d29f964372d8719d169fcecef641; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70955520, jitterRate=0.05731964111328125}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:41:01,728 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f666d29f964372d8719d169fcecef641 2024-12-11T22:41:01,728 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for f666d29f964372d8719d169fcecef641: Running coprocessor pre-open hook at 1733956861716Writing region info on filesystem at 1733956861716Initializing all the Stores at 1733956861717 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733956861717Cleaning up temporary data from old regions at 1733956861724 (+7 ms)Running coprocessor post-open hooks at 1733956861728 (+4 ms)Region opened successfully at 1733956861728 2024-12-11T22:41:01,729 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1733956859502.f666d29f964372d8719d169fcecef641., pid=59, masterSystemTime=1733956861706 2024-12-11T22:41:01,731 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1733956859502.f666d29f964372d8719d169fcecef641. 2024-12-11T22:41:01,731 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened test,mmm,1733956859502.f666d29f964372d8719d169fcecef641. 2024-12-11T22:41:01,732 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=f666d29f964372d8719d169fcecef641, regionState=OPEN, openSeqNum=5, regionLocation=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:01,735 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure f666d29f964372d8719d169fcecef641, server=56d7a8405f4a,34703,1733956857011 because future has completed 2024-12-11T22:41:01,739 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=57 2024-12-11T22:41:01,739 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=57, state=SUCCESS, hasLock=false; OpenRegionProcedure f666d29f964372d8719d169fcecef641, server=56d7a8405f4a,34703,1733956857011 in 185 msec 2024-12-11T22:41:01,741 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f666d29f964372d8719d169fcecef641, REOPEN/MOVE in 530 msec 2024-12-11T22:41:01,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-12-11T22:41:01,814 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T22:41:01,822 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.3 set balanceSwitch=false 2024-12-11T22:41:01,837 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=384, OpenFileDescriptor=643, MaxFileDescriptor=1048576, SystemLoadAverage=172, ProcessCount=11, AvailableMemoryMB=6075 2024-12-11T22:41:01,840 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-12-11T22:41:01,841 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-12-11T22:41:01,841 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 56d7a8405f4a,36971,1733956857186 2024-12-11T22:41:01,841 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 56d7a8405f4a,36971,1733956857186: Simulated kill ***** 2024-12-11T22:41:01,842 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-11T22:41:01,844 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-11T22:41:01,846 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-11T22:41:01,846 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-11T22:41:01,847 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-11T22:41:01,847 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 335449728 }, "NonHeapMemoryUsage": { "committed": 152436736, "init": 7667712, "max": -1, "used": 150120656 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "56d7a8405f4a" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "56d7a8405f4a" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "56d7a8405f4a" } ] } 2024-12-11T22:41:01,851 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34843 {}] master.MasterRpcServices(700): 56d7a8405f4a,36971,1733956857186 reported a fatal error: ***** ABORTING region server 56d7a8405f4a,36971,1733956857186: Simulated kill ***** 2024-12-11T22:41:01,853 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '56d7a8405f4a,36971,1733956857186' ***** 2024-12-11T22:41:01,854 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-12-11T22:41:01,854 INFO [RS:2;56d7a8405f4a:36971 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-11T22:41:01,854 INFO [RS:2;56d7a8405f4a:36971 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-11T22:41:01,854 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-11T22:41:01,854 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T22:41:01,854 INFO [RS:2;56d7a8405f4a:36971 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-11T22:41:01,854 INFO [RS:2;56d7a8405f4a:36971 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-11T22:41:01,855 INFO [RS:2;56d7a8405f4a:36971 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;56d7a8405f4a:36971. 2024-12-11T22:41:01,855 DEBUG [RS:2;56d7a8405f4a:36971 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T22:41:01,859 DEBUG [RS:2;56d7a8405f4a:36971 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T22:41:01,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741836_1012 (size=33749) 2024-12-11T22:41:01,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741836_1012 (size=33749) 2024-12-11T22:41:01,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741836_1012 (size=33749) 2024-12-11T22:41:01,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741835_1011 (size=2769) 2024-12-11T22:41:01,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741835_1011 (size=2769) 2024-12-11T22:41:01,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741835_1011 (size=2769) 2024-12-11T22:41:01,870 DEBUG [RS:2;56d7a8405f4a:36971 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T22:41:01,870 INFO [RS:2;56d7a8405f4a:36971 {}] regionserver.LeaseManager(133): Closed leases 2024-12-11T22:41:01,870 INFO [RS:2;56d7a8405f4a:36971 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36971 2024-12-11T22:41:01,880 INFO [regionserver/56d7a8405f4a:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-11T22:41:01,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36971-0x1001717a55e0003, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/56d7a8405f4a,36971,1733956857186 2024-12-11T22:41:01,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-11T22:41:01,930 INFO [RS:2;56d7a8405f4a:36971 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-11T22:41:01,932 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [56d7a8405f4a,36971,1733956857186] 2024-12-11T22:41:01,946 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/56d7a8405f4a,36971,1733956857186 already deleted, retry=false 2024-12-11T22:41:01,946 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 56d7a8405f4a,36971,1733956857186 on 56d7a8405f4a,34843,1733956856287 2024-12-11T22:41:01,950 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true 2024-12-11T22:41:01,955 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true 2024-12-11T22:41:01,955 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=60 for 56d7a8405f4a,36971,1733956857186 (carryingMeta=true) 56d7a8405f4a,36971,1733956857186/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@45310f07[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-11T22:41:01,956 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T22:41:01,957 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true, isMeta: true 2024-12-11T22:41:01,960 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,36971,1733956857186-splitting 2024-12-11T22:41:01,962 INFO [PEWorker-1 {}] master.SplitWALManager(105): 56d7a8405f4a,36971,1733956857186 WAL count=1, meta=true 2024-12-11T22:41:01,965 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta}] 2024-12-11T22:41:01,972 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:01,975 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131}] 2024-12-11T22:41:02,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36971-0x1001717a55e0003, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-11T22:41:02,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36971-0x1001717a55e0003, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-11T22:41:02,038 INFO [RS:2;56d7a8405f4a:36971 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-11T22:41:02,038 INFO [RS:2;56d7a8405f4a:36971 {}] regionserver.HRegionServer(1031): Exiting; stopping=56d7a8405f4a,36971,1733956857186; zookeeper connection closed. 2024-12-11T22:41:02,039 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@68311956 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@68311956 2024-12-11T22:41:02,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=62 2024-12-11T22:41:02,167 INFO [RS_LOG_REPLAY_OPS-regionserver/56d7a8405f4a:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(299): Splitting hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,36971,1733956857186-splitting/56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, size=33.0 K (33749bytes) 2024-12-11T22:41:02,167 DEBUG [RS_LOG_REPLAY_OPS-regionserver/56d7a8405f4a:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-11T22:41:02,168 DEBUG [RS_LOG_REPLAY_OPS-regionserver/56d7a8405f4a:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-11T22:41:02,168 INFO [RS_LOG_REPLAY_OPS-regionserver/56d7a8405f4a:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,36971,1733956857186-splitting/56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta 2024-12-11T22:41:02,172 INFO [RS_LOG_REPLAY_OPS-regionserver/56d7a8405f4a:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,36971,1733956857186-splitting/56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta after 3ms 2024-12-11T22:41:02,179 INFO [RS_LOG_REPLAY_OPS-regionserver/56d7a8405f4a:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(310): Open hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,36971,1733956857186-splitting/56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta took 13ms 2024-12-11T22:41:02,188 DEBUG [RS_LOG_REPLAY_OPS-regionserver/56d7a8405f4a:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-12-11T22:41:02,195 INFO [RS_LOG_REPLAY_OPS-regionserver/56d7a8405f4a:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta.temp 2024-12-11T22:41:02,197 INFO [RS_LOG_REPLAY_OPS-regionserver/56d7a8405f4a:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta.temp 2024-12-11T22:41:02,199 DEBUG [RS_LOG_REPLAY_OPS-regionserver/56d7a8405f4a:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,36971,1733956857186-splitting/56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta so closing down 2024-12-11T22:41:02,200 DEBUG [RS_LOG_REPLAY_OPS-regionserver/56d7a8405f4a:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-11T22:41:02,200 INFO [RS_LOG_REPLAY_OPS-regionserver/56d7a8405f4a:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(145): 3 split writer threads finished 2024-12-11T22:41:02,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741864_1040 (size=33508) 2024-12-11T22:41:02,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741864_1040 (size=33508) 2024-12-11T22:41:02,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741864_1040 (size=33508) 2024-12-11T22:41:02,211 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta.temp (wrote 61 edits, skipped 0 edits in 2 ms) 2024-12-11T22:41:02,214 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta.temp to hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-12-11T22:41:02,217 INFO [RS_LOG_REPLAY_OPS-regionserver/56d7a8405f4a:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(425): Processed 62 edits across 1 Regions in 33 ms; skipped=1; WAL=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,36971,1733956857186-splitting/56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false 2024-12-11T22:41:02,217 DEBUG [RS_LOG_REPLAY_OPS-regionserver/56d7a8405f4a:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(428): Completed split of hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,36971,1733956857186-splitting/56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, journal: Splitting hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,36971,1733956857186-splitting/56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, size=33.0 K (33749bytes) at 1733956862167Creating recovered edits writer path=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta.temp at 1733956862197 (+30 ms)Finishing writing output for hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,36971,1733956857186-splitting/56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta so closing down at 1733956862200 (+3 ms)3 split writer threads finished at 1733956862200Closed recovered edits writer path=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta.temp (wrote 61 edits, skipped 0 edits in 2 ms) at 1733956862211 (+11 ms)Rename recovered edits hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta.temp to hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 at 1733956862214 (+3 ms)Processed 62 edits across 1 Regions in 33 ms; skipped=1; WAL=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,36971,1733956857186-splitting/56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false at 1733956862217 (+3 ms) 2024-12-11T22:41:02,218 DEBUG [RS_LOG_REPLAY_OPS-regionserver/56d7a8405f4a:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,36971,1733956857186-splitting/56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta 2024-12-11T22:41:02,219 DEBUG [RS_LOG_REPLAY_OPS-regionserver/56d7a8405f4a:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=62 2024-12-11T22:41:02,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] master.HMaster(4169): Remote procedure done, pid=62 2024-12-11T22:41:02,228 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,36971,1733956857186-splitting/56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta to hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/oldWALs 2024-12-11T22:41:02,230 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-12-11T22:41:02,230 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 in 254 msec 2024-12-11T22:41:02,231 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:02,234 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-11T22:41:02,234 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 in 268 msec 2024-12-11T22:41:02,236 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,36971,1733956857186-splitting dir is empty, no logs to split. 2024-12-11T22:41:02,236 INFO [PEWorker-1 {}] master.SplitWALManager(105): 56d7a8405f4a,36971,1733956857186 WAL count=0, meta=true 2024-12-11T22:41:02,236 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 56d7a8405f4a,36971,1733956857186 WAL splitting is done? wals=0, meta=true 2024-12-11T22:41:02,239 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-11T22:41:02,240 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-11T22:41:02,241 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-11T22:41:02,391 DEBUG [56d7a8405f4a:34843 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-11T22:41:02,392 DEBUG [56d7a8405f4a:34843 {}] balancer.BalancerClusterState(204): Hosts are {56d7a8405f4a=0} racks are {/default-rack=0} 2024-12-11T22:41:02,392 DEBUG [56d7a8405f4a:34843 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-11T22:41:02,392 DEBUG [56d7a8405f4a:34843 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-11T22:41:02,392 DEBUG [56d7a8405f4a:34843 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-11T22:41:02,392 DEBUG [56d7a8405f4a:34843 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-11T22:41:02,392 INFO [56d7a8405f4a:34843 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-11T22:41:02,393 INFO [56d7a8405f4a:34843 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-11T22:41:02,393 DEBUG [56d7a8405f4a:34843 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-11T22:41:02,393 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:02,397 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 56d7a8405f4a,34703,1733956857011, state=OPENING 2024-12-11T22:41:02,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:02,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:02,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:02,429 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-11T22:41:02,429 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:02,429 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:02,429 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:02,430 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,34703,1733956857011}] 2024-12-11T22:41:02,500 INFO [regionserver/56d7a8405f4a:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-11T22:41:02,501 INFO [regionserver/56d7a8405f4a:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-11T22:41:02,590 INFO [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-11T22:41:02,590 INFO [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T22:41:02,591 INFO [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-11T22:41:02,593 INFO [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=56d7a8405f4a%2C34703%2C1733956857011.meta, suffix=.meta, logDir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,34703,1733956857011, archiveDir=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/oldWALs, maxLogs=32 2024-12-11T22:41:02,610 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,34703,1733956857011/56d7a8405f4a%2C34703%2C1733956857011.meta.1733956862595.meta, exclude list is [], retry=0 2024-12-11T22:41:02,613 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33033,DS-f86c6cf2-47aa-4d2f-ba7f-902e5eaa2afe,DISK] 2024-12-11T22:41:02,613 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42009,DS-34028945-25f0-40a0-8da9-6b2be7f53a5e,DISK] 2024-12-11T22:41:02,613 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36811,DS-018fb24c-ac8f-4f8e-ace7-2b9f38ac5619,DISK] 2024-12-11T22:41:02,615 INFO [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,34703,1733956857011/56d7a8405f4a%2C34703%2C1733956857011.meta.1733956862595.meta 2024-12-11T22:41:02,615 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46615:46615),(127.0.0.1/127.0.0.1:43991:43991),(127.0.0.1/127.0.0.1:44105:44105)] 2024-12-11T22:41:02,615 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-11T22:41:02,616 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-11T22:41:02,616 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-11T22:41:02,616 INFO [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-11T22:41:02,616 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-11T22:41:02,616 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:41:02,616 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-11T22:41:02,616 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-11T22:41:02,618 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-11T22:41:02,618 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-11T22:41:02,619 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:41:02,619 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T22:41:02,619 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-11T22:41:02,620 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-11T22:41:02,620 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:41:02,621 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T22:41:02,621 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-11T22:41:02,622 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-11T22:41:02,622 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:41:02,622 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T22:41:02,622 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-11T22:41:02,623 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-11T22:41:02,623 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:41:02,624 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T22:41:02,624 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-11T22:41:02,625 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/hbase/meta/1588230740 2024-12-11T22:41:02,627 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/hbase/meta/1588230740 2024-12-11T22:41:02,629 INFO [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-12-11T22:41:02,643 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5793): Applied 277, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=64, path=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-12-11T22:41:02,646 INFO [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=29.41 KB heapSize=50.67 KB 2024-12-11T22:41:02,702 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/hbase/meta/1588230740/.tmp/info/39a6d4a511cd40168b7b32000de020f3 is 138, key is test,bbb,1733956859502.66aac9e74e8263b0b8914fdc86eec66c./info:regioninfo/1733956860325/Put/seqid=0 2024-12-11T22:41:02,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741866_1042 (size=36381) 2024-12-11T22:41:02,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741866_1042 (size=36381) 2024-12-11T22:41:02,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741866_1042 (size=36381) 2024-12-11T22:41:02,722 INFO [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.26 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/hbase/meta/1588230740/.tmp/info/39a6d4a511cd40168b7b32000de020f3 2024-12-11T22:41:02,788 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/hbase/meta/1588230740/.tmp/ns/e4d59ca894b64a73b607cf2c697d3cbc is 43, key is default/ns:d/1733956859288/Put/seqid=0 2024-12-11T22:41:02,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741867_1043 (size=5153) 2024-12-11T22:41:02,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741867_1043 (size=5153) 2024-12-11T22:41:02,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741867_1043 (size=5153) 2024-12-11T22:41:02,799 INFO [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/hbase/meta/1588230740/.tmp/ns/e4d59ca894b64a73b607cf2c697d3cbc 2024-12-11T22:41:02,824 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/hbase/meta/1588230740/.tmp/table/c24ceed9f5264e88817f08a526a008c4 is 40, key is test/table:state/1733956860584/Put/seqid=0 2024-12-11T22:41:02,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741868_1044 (size=5165) 2024-12-11T22:41:02,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741868_1044 (size=5165) 2024-12-11T22:41:02,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741868_1044 (size=5165) 2024-12-11T22:41:02,833 INFO [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/hbase/meta/1588230740/.tmp/table/c24ceed9f5264e88817f08a526a008c4 2024-12-11T22:41:02,844 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/hbase/meta/1588230740/.tmp/info/39a6d4a511cd40168b7b32000de020f3 as hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/hbase/meta/1588230740/info/39a6d4a511cd40168b7b32000de020f3 2024-12-11T22:41:02,854 INFO [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/hbase/meta/1588230740/info/39a6d4a511cd40168b7b32000de020f3, entries=273, sequenceid=64, filesize=35.5 K 2024-12-11T22:41:02,857 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/hbase/meta/1588230740/.tmp/ns/e4d59ca894b64a73b607cf2c697d3cbc as hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/hbase/meta/1588230740/ns/e4d59ca894b64a73b607cf2c697d3cbc 2024-12-11T22:41:02,866 INFO [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/hbase/meta/1588230740/ns/e4d59ca894b64a73b607cf2c697d3cbc, entries=2, sequenceid=64, filesize=5.0 K 2024-12-11T22:41:02,868 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/hbase/meta/1588230740/.tmp/table/c24ceed9f5264e88817f08a526a008c4 as hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/hbase/meta/1588230740/table/c24ceed9f5264e88817f08a526a008c4 2024-12-11T22:41:02,877 INFO [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/hbase/meta/1588230740/table/c24ceed9f5264e88817f08a526a008c4, entries=2, sequenceid=64, filesize=5.0 K 2024-12-11T22:41:02,883 INFO [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(3140): Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 235ms, sequenceid=64, compaction requested=false; wal=null 2024-12-11T22:41:02,884 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-11T22:41:02,888 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-12-11T22:41:02,890 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-11T22:41:02,890 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-11T22:41:02,891 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-11T22:41:02,893 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-11T22:41:02,896 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/hbase/meta/1588230740/recovered.edits/64.seqid, newMaxSeqId=64, maxSeqId=1 2024-12-11T22:41:02,898 INFO [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=65; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66760116, jitterRate=-0.005196750164031982}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-11T22:41:02,898 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-11T22:41:02,898 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733956862616Writing region info on filesystem at 1733956862616Initializing all the Stores at 1733956862617 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733956862617Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733956862617Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733956862617Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733956862617Obtaining lock to block concurrent updates at 1733956862646 (+29 ms)Preparing flush snapshotting stores in 1588230740 at 1733956862646Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=30111, getHeapSize=51824, getOffHeapSize=0, getCellsCount=277 at 1733956862648 (+2 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733956862648Flushing 1588230740/info: creating writer at 1733956862649 (+1 ms)Flushing 1588230740/info: appending metadata at 1733956862691 (+42 ms)Flushing 1588230740/info: closing flushed file at 1733956862694 (+3 ms)Flushing 1588230740/ns: creating writer at 1733956862766 (+72 ms)Flushing 1588230740/ns: appending metadata at 1733956862786 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1733956862786Flushing 1588230740/table: creating writer at 1733956862808 (+22 ms)Flushing 1588230740/table: appending metadata at 1733956862823 (+15 ms)Flushing 1588230740/table: closing flushed file at 1733956862823Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f57230d: reopening flushed file at 1733956862843 (+20 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36a6b3e0: reopening flushed file at 1733956862856 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42c20f71: reopening flushed file at 1733956862866 (+10 ms)Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 235ms, sequenceid=64, compaction requested=false; wal=null at 1733956862883 (+17 ms)Cleaning up temporary data from old regions at 1733956862890 (+7 ms)Running coprocessor post-open hooks at 1733956862898 (+8 ms)Region opened successfully at 1733956862898 2024-12-11T22:41:02,900 INFO [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=64, masterSystemTime=1733956862584 2024-12-11T22:41:02,902 DEBUG [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-11T22:41:02,902 INFO [RS_OPEN_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-11T22:41:02,903 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=65, regionLocation=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:02,904 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 56d7a8405f4a,34703,1733956857011, state=OPEN 2024-12-11T22:41:02,920 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34843 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=56d7a8405f4a,34703,1733956857011, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-11T22:41:02,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:02,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:02,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:02,921 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:02,921 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:02,921 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:02,921 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=64, ppid=63, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:02,924 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-12-11T22:41:02,924 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,34703,1733956857011 in 492 msec 2024-12-11T22:41:02,926 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=60 2024-12-11T22:41:02,926 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 686 msec 2024-12-11T22:41:02,927 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 56d7a8405f4a,36971,1733956857186 had 9 regions 2024-12-11T22:41:02,928 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true, isMeta: false 2024-12-11T22:41:02,930 INFO [PEWorker-1 {}] master.SplitWALManager(105): 56d7a8405f4a,36971,1733956857186 WAL count=1, meta=false 2024-12-11T22:41:02,931 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632}] 2024-12-11T22:41:02,933 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:02,934 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131}] 2024-12-11T22:41:03,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=66 2024-12-11T22:41:03,103 INFO [RS_LOG_REPLAY_OPS-regionserver/56d7a8405f4a:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(299): Splitting hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,36971,1733956857186-splitting/56d7a8405f4a%2C36971%2C1733956857186.1733956858632, size=2.7 K (2769bytes) 2024-12-11T22:41:03,103 INFO [RS_LOG_REPLAY_OPS-regionserver/56d7a8405f4a:0-1 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,36971,1733956857186-splitting/56d7a8405f4a%2C36971%2C1733956857186.1733956858632 2024-12-11T22:41:03,104 INFO [RS_LOG_REPLAY_OPS-regionserver/56d7a8405f4a:0-1 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,36971,1733956857186-splitting/56d7a8405f4a%2C36971%2C1733956857186.1733956858632 after 1ms 2024-12-11T22:41:03,107 INFO [RS_LOG_REPLAY_OPS-regionserver/56d7a8405f4a:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(310): Open hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,36971,1733956857186-splitting/56d7a8405f4a%2C36971%2C1733956857186.1733956858632 took 4ms 2024-12-11T22:41:03,110 DEBUG [RS_LOG_REPLAY_OPS-regionserver/56d7a8405f4a:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for b30c1b01a8241f23347f03f93df7f090: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-11T22:41:03,111 DEBUG [RS_LOG_REPLAY_OPS-regionserver/56d7a8405f4a:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for de3803c768bc063bdc71e495af58eb48: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-11T22:41:03,113 DEBUG [RS_LOG_REPLAY_OPS-regionserver/56d7a8405f4a:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for ad4bb3db7f3f05f24b6115fc887d6ab5: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-11T22:41:03,115 DEBUG [RS_LOG_REPLAY_OPS-regionserver/56d7a8405f4a:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for a0e02a7a4f8d57169d1f435eabc00839: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-11T22:41:03,116 DEBUG [RS_LOG_REPLAY_OPS-regionserver/56d7a8405f4a:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for f666d29f964372d8719d169fcecef641: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-11T22:41:03,118 DEBUG [RS_LOG_REPLAY_OPS-regionserver/56d7a8405f4a:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for f219e06826a4a3ede110adacdf28f6f4: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-11T22:41:03,120 DEBUG [RS_LOG_REPLAY_OPS-regionserver/56d7a8405f4a:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 5a93ec512c97cf46e8942b91f4c8f088: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-11T22:41:03,121 DEBUG [RS_LOG_REPLAY_OPS-regionserver/56d7a8405f4a:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 01df3c1c862de5abd063d235e117a115: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-11T22:41:03,122 DEBUG [RS_LOG_REPLAY_OPS-regionserver/56d7a8405f4a:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 419f7368f61981f0a715c20304b74e0f: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-11T22:41:03,123 DEBUG [RS_LOG_REPLAY_OPS-regionserver/56d7a8405f4a:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,36971,1733956857186-splitting/56d7a8405f4a%2C36971%2C1733956857186.1733956858632 so closing down 2024-12-11T22:41:03,123 DEBUG [RS_LOG_REPLAY_OPS-regionserver/56d7a8405f4a:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-11T22:41:03,123 INFO [RS_LOG_REPLAY_OPS-regionserver/56d7a8405f4a:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(145): 3 split writer threads finished 2024-12-11T22:41:03,123 INFO [RS_LOG_REPLAY_OPS-regionserver/56d7a8405f4a:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(425): Processed 10 edits across 0 Regions in 16 ms; skipped=10; WAL=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,36971,1733956857186-splitting/56d7a8405f4a%2C36971%2C1733956857186.1733956858632, size=2.7 K, length=2769, corrupted=false, cancelled=false 2024-12-11T22:41:03,123 DEBUG [RS_LOG_REPLAY_OPS-regionserver/56d7a8405f4a:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(428): Completed split of hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,36971,1733956857186-splitting/56d7a8405f4a%2C36971%2C1733956857186.1733956858632, journal: Splitting hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,36971,1733956857186-splitting/56d7a8405f4a%2C36971%2C1733956857186.1733956858632, size=2.7 K (2769bytes) at 1733956863103Finishing writing output for hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,36971,1733956857186-splitting/56d7a8405f4a%2C36971%2C1733956857186.1733956858632 so closing down at 1733956863123 (+20 ms)3 split writer threads finished at 1733956863123Processed 10 edits across 0 Regions in 16 ms; skipped=10; WAL=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,36971,1733956857186-splitting/56d7a8405f4a%2C36971%2C1733956857186.1733956858632, size=2.7 K, length=2769, corrupted=false, cancelled=false at 1733956863123 2024-12-11T22:41:03,123 DEBUG [RS_LOG_REPLAY_OPS-regionserver/56d7a8405f4a:0-1 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,36971,1733956857186-splitting/56d7a8405f4a%2C36971%2C1733956857186.1733956858632 2024-12-11T22:41:03,123 DEBUG [RS_LOG_REPLAY_OPS-regionserver/56d7a8405f4a:0-1 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=66 2024-12-11T22:41:03,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34843 {}] master.HMaster(4169): Remote procedure done, pid=66 2024-12-11T22:41:03,128 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,36971,1733956857186-splitting/56d7a8405f4a%2C36971%2C1733956857186.1733956858632 to hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/oldWALs 2024-12-11T22:41:03,130 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-12-11T22:41:03,131 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 in 194 msec 2024-12-11T22:41:03,132 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,136 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=60 2024-12-11T22:41:03,136 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 in 202 msec 2024-12-11T22:41:03,138 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/WALs/56d7a8405f4a,36971,1733956857186-splitting dir is empty, no logs to split. 2024-12-11T22:41:03,138 INFO [PEWorker-1 {}] master.SplitWALManager(105): 56d7a8405f4a,36971,1733956857186 WAL count=0, meta=false 2024-12-11T22:41:03,138 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 56d7a8405f4a,36971,1733956857186 WAL splitting is done? wals=0, meta=false 2024-12-11T22:41:03,141 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN}, {pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN}, {pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN}, {pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN}, {pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN}, {pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN}, {pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN}, {pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN}] 2024-12-11T22:41:03,143 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN 2024-12-11T22:41:03,144 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-11T22:41:03,146 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN 2024-12-11T22:41:03,147 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-11T22:41:03,148 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN 2024-12-11T22:41:03,149 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-11T22:41:03,149 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN 2024-12-11T22:41:03,151 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-11T22:41:03,152 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN 2024-12-11T22:41:03,153 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-11T22:41:03,154 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN 2024-12-11T22:41:03,155 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-11T22:41:03,156 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN 2024-12-11T22:41:03,157 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-11T22:41:03,157 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN 2024-12-11T22:41:03,158 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-11T22:41:03,295 DEBUG [56d7a8405f4a:34843 {}] balancer.BalancerClusterState(204): Hosts are {56d7a8405f4a=0} racks are {/default-rack=0} 2024-12-11T22:41:03,295 DEBUG [56d7a8405f4a:34843 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-11T22:41:03,295 DEBUG [56d7a8405f4a:34843 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-11T22:41:03,295 DEBUG [56d7a8405f4a:34843 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-11T22:41:03,295 DEBUG [56d7a8405f4a:34843 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-11T22:41:03,295 INFO [56d7a8405f4a:34843 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-11T22:41:03,295 INFO [56d7a8405f4a:34843 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-11T22:41:03,295 DEBUG [56d7a8405f4a:34843 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-11T22:41:03,296 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=b30c1b01a8241f23347f03f93df7f090, regionState=OPENING, regionLocation=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:03,296 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=73 updating hbase:meta row=01df3c1c862de5abd063d235e117a115, regionState=OPENING, regionLocation=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:03,297 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=de3803c768bc063bdc71e495af58eb48, regionState=OPENING, regionLocation=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:03,298 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=5a93ec512c97cf46e8942b91f4c8f088, regionState=OPENING, regionLocation=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:03,297 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 56d7a8405f4a:36971 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 56d7a8405f4a/172.17.0.3:36971 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T22:41:03,298 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=a0e02a7a4f8d57169d1f435eabc00839, regionState=OPENING, regionLocation=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,299 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=f219e06826a4a3ede110adacdf28f6f4, regionState=OPENING, regionLocation=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,299 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=ad4bb3db7f3f05f24b6115fc887d6ab5, regionState=OPENING, regionLocation=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,299 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=56d7a8405f4a,36971,1733956857186, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=56d7a8405f4a,36971,1733956857186, seqNum=-1, error=java.net.ConnectException: Call to address=56d7a8405f4a:36971 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 56d7a8405f4a/172.17.0.3:36971 2024-12-11T22:41:03,300 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=419f7368f61981f0a715c20304b74e0f, regionState=OPENING, regionLocation=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,300 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=56d7a8405f4a,36971,1733956857186, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-12-11T22:41:03,300 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=56d7a8405f4a,36971,1733956857186, seqNum=-1 from cache 2024-12-11T22:41:03,301 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 56d7a8405f4a:36971 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 56d7a8405f4a/172.17.0.3:36971 2024-12-11T22:41:03,301 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 56d7a8405f4a:36971 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 56d7a8405f4a/172.17.0.3:36971 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T22:41:03,302 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=56d7a8405f4a,36971,1733956857186, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=56d7a8405f4a:36971 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 56d7a8405f4a/172.17.0.3:36971 2024-12-11T22:41:03,302 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=56d7a8405f4a,36971,1733956857186, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=56d7a8405f4a:36971 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 56d7a8405f4a/172.17.0.3:36971 2024-12-11T22:41:03,302 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=56d7a8405f4a,36971,1733956857186, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=56d7a8405f4a:36971 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 56d7a8405f4a/172.17.0.3:36971 2024-12-11T22:41:03,302 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=56d7a8405f4a,36971,1733956857186, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=56d7a8405f4a:36971 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 56d7a8405f4a/172.17.0.3:36971 2024-12-11T22:41:03,302 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=56d7a8405f4a,36971,1733956857186, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=56d7a8405f4a:36971 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 56d7a8405f4a/172.17.0.3:36971 2024-12-11T22:41:03,302 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=56d7a8405f4a,36971,1733956857186, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=56d7a8405f4a:36971 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 56d7a8405f4a/172.17.0.3:36971 2024-12-11T22:41:03,302 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=56d7a8405f4a,36971,1733956857186, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=56d7a8405f4a:36971 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 56d7a8405f4a/172.17.0.3:36971 2024-12-11T22:41:03,303 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 56d7a8405f4a:36971 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 56d7a8405f4a/172.17.0.3:36971 2024-12-11T22:41:03,405 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T22:41:03,405 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=56d7a8405f4a,34703,1733956857011, seqNum=-1] 2024-12-11T22:41:03,405 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T22:41:03,407 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37899, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T22:41:03,410 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN because future has completed 2024-12-11T22:41:03,411 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure b30c1b01a8241f23347f03f93df7f090, server=56d7a8405f4a,34703,1733956857011}] 2024-12-11T22:41:03,412 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN because future has completed 2024-12-11T22:41:03,413 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN because future has completed 2024-12-11T22:41:03,414 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN because future has completed 2024-12-11T22:41:03,415 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN because future has completed 2024-12-11T22:41:03,416 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN because future has completed 2024-12-11T22:41:03,417 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN because future has completed 2024-12-11T22:41:03,418 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN because future has completed 2024-12-11T22:41:03,437 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure f219e06826a4a3ede110adacdf28f6f4, server=56d7a8405f4a,40421,1733956857131}] 2024-12-11T22:41:03,442 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0e02a7a4f8d57169d1f435eabc00839, server=56d7a8405f4a,40421,1733956857131}] 2024-12-11T22:41:03,446 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011}] 2024-12-11T22:41:03,453 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure de3803c768bc063bdc71e495af58eb48, server=56d7a8405f4a,34703,1733956857011}] 2024-12-11T22:41:03,456 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 01df3c1c862de5abd063d235e117a115, server=56d7a8405f4a,34703,1733956857011}] 2024-12-11T22:41:03,460 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 419f7368f61981f0a715c20304b74e0f, server=56d7a8405f4a,40421,1733956857131}] 2024-12-11T22:41:03,463 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure ad4bb3db7f3f05f24b6115fc887d6ab5, server=56d7a8405f4a,40421,1733956857131}] 2024-12-11T22:41:03,590 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,,1733956859502.5a93ec512c97cf46e8942b91f4c8f088. 2024-12-11T22:41:03,590 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => 5a93ec512c97cf46e8942b91f4c8f088, NAME => 'test,,1733956859502.5a93ec512c97cf46e8942b91f4c8f088.', STARTKEY => '', ENDKEY => 'bbb'} 2024-12-11T22:41:03,591 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5a93ec512c97cf46e8942b91f4c8f088 2024-12-11T22:41:03,591 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,,1733956859502.5a93ec512c97cf46e8942b91f4c8f088.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:41:03,591 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for 5a93ec512c97cf46e8942b91f4c8f088 2024-12-11T22:41:03,591 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for 5a93ec512c97cf46e8942b91f4c8f088 2024-12-11T22:41:03,593 INFO [StoreOpener-5a93ec512c97cf46e8942b91f4c8f088-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5a93ec512c97cf46e8942b91f4c8f088 2024-12-11T22:41:03,595 INFO [StoreOpener-5a93ec512c97cf46e8942b91f4c8f088-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a93ec512c97cf46e8942b91f4c8f088 columnFamilyName family 2024-12-11T22:41:03,595 DEBUG [StoreOpener-5a93ec512c97cf46e8942b91f4c8f088-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:41:03,595 INFO [StoreOpener-5a93ec512c97cf46e8942b91f4c8f088-1 {}] regionserver.HStore(327): Store=5a93ec512c97cf46e8942b91f4c8f088/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:41:03,596 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,xxx,1733956859502.a0e02a7a4f8d57169d1f435eabc00839. 2024-12-11T22:41:03,596 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for 5a93ec512c97cf46e8942b91f4c8f088 2024-12-11T22:41:03,596 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => a0e02a7a4f8d57169d1f435eabc00839, NAME => 'test,xxx,1733956859502.a0e02a7a4f8d57169d1f435eabc00839.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-12-11T22:41:03,596 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a0e02a7a4f8d57169d1f435eabc00839 2024-12-11T22:41:03,597 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,xxx,1733956859502.a0e02a7a4f8d57169d1f435eabc00839.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:41:03,597 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for a0e02a7a4f8d57169d1f435eabc00839 2024-12-11T22:41:03,597 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for a0e02a7a4f8d57169d1f435eabc00839 2024-12-11T22:41:03,597 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/5a93ec512c97cf46e8942b91f4c8f088 2024-12-11T22:41:03,599 INFO [StoreOpener-a0e02a7a4f8d57169d1f435eabc00839-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a0e02a7a4f8d57169d1f435eabc00839 2024-12-11T22:41:03,599 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/5a93ec512c97cf46e8942b91f4c8f088 2024-12-11T22:41:03,599 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for 5a93ec512c97cf46e8942b91f4c8f088 2024-12-11T22:41:03,600 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for 5a93ec512c97cf46e8942b91f4c8f088 2024-12-11T22:41:03,600 INFO [StoreOpener-a0e02a7a4f8d57169d1f435eabc00839-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a0e02a7a4f8d57169d1f435eabc00839 columnFamilyName family 2024-12-11T22:41:03,600 DEBUG [StoreOpener-a0e02a7a4f8d57169d1f435eabc00839-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:41:03,601 INFO [StoreOpener-a0e02a7a4f8d57169d1f435eabc00839-1 {}] regionserver.HStore(327): Store=a0e02a7a4f8d57169d1f435eabc00839/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:41:03,601 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for a0e02a7a4f8d57169d1f435eabc00839 2024-12-11T22:41:03,602 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/a0e02a7a4f8d57169d1f435eabc00839 2024-12-11T22:41:03,602 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for 5a93ec512c97cf46e8942b91f4c8f088 2024-12-11T22:41:03,602 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened 5a93ec512c97cf46e8942b91f4c8f088; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71088903, jitterRate=0.05930720269680023}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:41:03,602 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5a93ec512c97cf46e8942b91f4c8f088 2024-12-11T22:41:03,603 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for 5a93ec512c97cf46e8942b91f4c8f088: Running coprocessor pre-open hook at 1733956863591Writing region info on filesystem at 1733956863591Initializing all the Stores at 1733956863592 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733956863592Cleaning up temporary data from old regions at 1733956863600 (+8 ms)Running coprocessor post-open hooks at 1733956863602 (+2 ms)Region opened successfully at 1733956863603 (+1 ms) 2024-12-11T22:41:03,603 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/a0e02a7a4f8d57169d1f435eabc00839 2024-12-11T22:41:03,603 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for a0e02a7a4f8d57169d1f435eabc00839 2024-12-11T22:41:03,603 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for a0e02a7a4f8d57169d1f435eabc00839 2024-12-11T22:41:03,603 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1733956859502.5a93ec512c97cf46e8942b91f4c8f088., pid=78, masterSystemTime=1733956863585 2024-12-11T22:41:03,605 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for a0e02a7a4f8d57169d1f435eabc00839 2024-12-11T22:41:03,606 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1733956859502.5a93ec512c97cf46e8942b91f4c8f088. 2024-12-11T22:41:03,606 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened test,,1733956859502.5a93ec512c97cf46e8942b91f4c8f088. 2024-12-11T22:41:03,606 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,vvv,1733956859502.b30c1b01a8241f23347f03f93df7f090. 2024-12-11T22:41:03,606 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => b30c1b01a8241f23347f03f93df7f090, NAME => 'test,vvv,1733956859502.b30c1b01a8241f23347f03f93df7f090.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-12-11T22:41:03,607 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b30c1b01a8241f23347f03f93df7f090 2024-12-11T22:41:03,607 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened a0e02a7a4f8d57169d1f435eabc00839; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70436263, jitterRate=0.0495821088552475}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:41:03,607 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,vvv,1733956859502.b30c1b01a8241f23347f03f93df7f090.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:41:03,607 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a0e02a7a4f8d57169d1f435eabc00839 2024-12-11T22:41:03,607 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for b30c1b01a8241f23347f03f93df7f090 2024-12-11T22:41:03,607 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for a0e02a7a4f8d57169d1f435eabc00839: Running coprocessor pre-open hook at 1733956863597Writing region info on filesystem at 1733956863597Initializing all the Stores at 1733956863598 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733956863598Cleaning up temporary data from old regions at 1733956863603 (+5 ms)Running coprocessor post-open hooks at 1733956863607 (+4 ms)Region opened successfully at 1733956863607 2024-12-11T22:41:03,607 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for b30c1b01a8241f23347f03f93df7f090 2024-12-11T22:41:03,608 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1733956859502.a0e02a7a4f8d57169d1f435eabc00839., pid=77, masterSystemTime=1733956863590 2024-12-11T22:41:03,608 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-12-11T22:41:03,609 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:03,609 INFO [StoreOpener-b30c1b01a8241f23347f03f93df7f090-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b30c1b01a8241f23347f03f93df7f090 2024-12-11T22:41:03,609 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:03,610 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1733956859502.a0e02a7a4f8d57169d1f435eabc00839. 2024-12-11T22:41:03,610 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened test,xxx,1733956859502.a0e02a7a4f8d57169d1f435eabc00839. 2024-12-11T22:41:03,610 INFO [StoreOpener-b30c1b01a8241f23347f03f93df7f090-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b30c1b01a8241f23347f03f93df7f090 columnFamilyName family 2024-12-11T22:41:03,610 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(132): Open test,jjj,1733956859502.ad4bb3db7f3f05f24b6115fc887d6ab5. 2024-12-11T22:41:03,611 DEBUG [StoreOpener-b30c1b01a8241f23347f03f93df7f090-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:41:03,611 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7752): Opening region: {ENCODED => ad4bb3db7f3f05f24b6115fc887d6ab5, NAME => 'test,jjj,1733956859502.ad4bb3db7f3f05f24b6115fc887d6ab5.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-12-11T22:41:03,611 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ad4bb3db7f3f05f24b6115fc887d6ab5 2024-12-11T22:41:03,611 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(898): Instantiated test,jjj,1733956859502.ad4bb3db7f3f05f24b6115fc887d6ab5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:41:03,611 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7794): checking encryption for ad4bb3db7f3f05f24b6115fc887d6ab5 2024-12-11T22:41:03,611 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7797): checking classloading for ad4bb3db7f3f05f24b6115fc887d6ab5 2024-12-11T22:41:03,611 INFO [StoreOpener-b30c1b01a8241f23347f03f93df7f090-1 {}] regionserver.HStore(327): Store=b30c1b01a8241f23347f03f93df7f090/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:41:03,612 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for b30c1b01a8241f23347f03f93df7f090 2024-12-11T22:41:03,612 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/b30c1b01a8241f23347f03f93df7f090 2024-12-11T22:41:03,613 INFO [StoreOpener-ad4bb3db7f3f05f24b6115fc887d6ab5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ad4bb3db7f3f05f24b6115fc887d6ab5 2024-12-11T22:41:03,613 INFO [StoreOpener-ad4bb3db7f3f05f24b6115fc887d6ab5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ad4bb3db7f3f05f24b6115fc887d6ab5 columnFamilyName family 2024-12-11T22:41:03,613 DEBUG [StoreOpener-ad4bb3db7f3f05f24b6115fc887d6ab5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:41:03,614 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/b30c1b01a8241f23347f03f93df7f090 2024-12-11T22:41:03,614 INFO [StoreOpener-ad4bb3db7f3f05f24b6115fc887d6ab5-1 {}] regionserver.HStore(327): Store=ad4bb3db7f3f05f24b6115fc887d6ab5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:41:03,614 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for b30c1b01a8241f23347f03f93df7f090 2024-12-11T22:41:03,614 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for b30c1b01a8241f23347f03f93df7f090 2024-12-11T22:41:03,614 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1038): replaying wal for ad4bb3db7f3f05f24b6115fc887d6ab5 2024-12-11T22:41:03,615 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/ad4bb3db7f3f05f24b6115fc887d6ab5 2024-12-11T22:41:03,616 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/ad4bb3db7f3f05f24b6115fc887d6ab5 2024-12-11T22:41:03,617 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for b30c1b01a8241f23347f03f93df7f090 2024-12-11T22:41:03,617 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1048): stopping wal replay for ad4bb3db7f3f05f24b6115fc887d6ab5 2024-12-11T22:41:03,617 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1060): Cleaning up temporary data for ad4bb3db7f3f05f24b6115fc887d6ab5 2024-12-11T22:41:03,619 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened b30c1b01a8241f23347f03f93df7f090; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71838580, jitterRate=0.07047826051712036}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:41:03,619 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b30c1b01a8241f23347f03f93df7f090 2024-12-11T22:41:03,619 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for b30c1b01a8241f23347f03f93df7f090: Running coprocessor pre-open hook at 1733956863607Writing region info on filesystem at 1733956863607Initializing all the Stores at 1733956863608 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733956863608Cleaning up temporary data from old regions at 1733956863614 (+6 ms)Running coprocessor post-open hooks at 1733956863619 (+5 ms)Region opened successfully at 1733956863619 2024-12-11T22:41:03,619 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1093): writing seq id for ad4bb3db7f3f05f24b6115fc887d6ab5 2024-12-11T22:41:03,620 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1733956859502.b30c1b01a8241f23347f03f93df7f090., pid=75, masterSystemTime=1733956863585 2024-12-11T22:41:03,619 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 4 more 2024-12-11T22:41:03,621 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-11T22:41:03,621 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=82, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure ad4bb3db7f3f05f24b6115fc887d6ab5, server=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,621 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:03,621 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1114): Opened ad4bb3db7f3f05f24b6115fc887d6ab5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72869962, jitterRate=0.08584704995155334}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:41:03,622 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ad4bb3db7f3f05f24b6115fc887d6ab5 2024-12-11T22:41:03,622 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1006): Region open journal for ad4bb3db7f3f05f24b6115fc887d6ab5: Running coprocessor pre-open hook at 1733956863611Writing region info on filesystem at 1733956863611Initializing all the Stores at 1733956863612 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733956863612Cleaning up temporary data from old regions at 1733956863617 (+5 ms)Running coprocessor post-open hooks at 1733956863622 (+5 ms)Region opened successfully at 1733956863622 2024-12-11T22:41:03,622 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=82, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure ad4bb3db7f3f05f24b6115fc887d6ab5, server=56d7a8405f4a,40421,1733956857131 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:03,623 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1733956859502.ad4bb3db7f3f05f24b6115fc887d6ab5., pid=82, masterSystemTime=1733956863590 2024-12-11T22:41:03,624 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1733956859502.b30c1b01a8241f23347f03f93df7f090. 2024-12-11T22:41:03,624 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,vvv,1733956859502.b30c1b01a8241f23347f03f93df7f090. 2024-12-11T22:41:03,624 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,ggg,1733956859502.de3803c768bc063bdc71e495af58eb48. 2024-12-11T22:41:03,624 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => de3803c768bc063bdc71e495af58eb48, NAME => 'test,ggg,1733956859502.de3803c768bc063bdc71e495af58eb48.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-12-11T22:41:03,624 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test de3803c768bc063bdc71e495af58eb48 2024-12-11T22:41:03,624 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,ggg,1733956859502.de3803c768bc063bdc71e495af58eb48.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:41:03,624 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for de3803c768bc063bdc71e495af58eb48 2024-12-11T22:41:03,624 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for de3803c768bc063bdc71e495af58eb48 2024-12-11T22:41:03,626 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1733956859502.ad4bb3db7f3f05f24b6115fc887d6ab5. 2024-12-11T22:41:03,626 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(153): Opened test,jjj,1733956859502.ad4bb3db7f3f05f24b6115fc887d6ab5. 2024-12-11T22:41:03,626 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open test,fff,1733956859502.419f7368f61981f0a715c20304b74e0f. 2024-12-11T22:41:03,626 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => 419f7368f61981f0a715c20304b74e0f, NAME => 'test,fff,1733956859502.419f7368f61981f0a715c20304b74e0f.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-12-11T22:41:03,626 INFO [StoreOpener-de3803c768bc063bdc71e495af58eb48-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region de3803c768bc063bdc71e495af58eb48 2024-12-11T22:41:03,626 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 419f7368f61981f0a715c20304b74e0f 2024-12-11T22:41:03,627 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated test,fff,1733956859502.419f7368f61981f0a715c20304b74e0f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:41:03,627 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for 419f7368f61981f0a715c20304b74e0f 2024-12-11T22:41:03,627 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for 419f7368f61981f0a715c20304b74e0f 2024-12-11T22:41:03,628 INFO [StoreOpener-de3803c768bc063bdc71e495af58eb48-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de3803c768bc063bdc71e495af58eb48 columnFamilyName family 2024-12-11T22:41:03,628 DEBUG [StoreOpener-de3803c768bc063bdc71e495af58eb48-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:41:03,628 INFO [StoreOpener-419f7368f61981f0a715c20304b74e0f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 419f7368f61981f0a715c20304b74e0f 2024-12-11T22:41:03,628 INFO [StoreOpener-de3803c768bc063bdc71e495af58eb48-1 {}] regionserver.HStore(327): Store=de3803c768bc063bdc71e495af58eb48/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:41:03,628 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for de3803c768bc063bdc71e495af58eb48 2024-12-11T22:41:03,629 INFO [StoreOpener-419f7368f61981f0a715c20304b74e0f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 419f7368f61981f0a715c20304b74e0f columnFamilyName family 2024-12-11T22:41:03,629 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/de3803c768bc063bdc71e495af58eb48 2024-12-11T22:41:03,629 DEBUG [StoreOpener-419f7368f61981f0a715c20304b74e0f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:41:03,629 INFO [StoreOpener-419f7368f61981f0a715c20304b74e0f-1 {}] regionserver.HStore(327): Store=419f7368f61981f0a715c20304b74e0f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:41:03,630 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for 419f7368f61981f0a715c20304b74e0f 2024-12-11T22:41:03,630 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/de3803c768bc063bdc71e495af58eb48 2024-12-11T22:41:03,630 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/419f7368f61981f0a715c20304b74e0f 2024-12-11T22:41:03,631 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for de3803c768bc063bdc71e495af58eb48 2024-12-11T22:41:03,631 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for de3803c768bc063bdc71e495af58eb48 2024-12-11T22:41:03,631 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/419f7368f61981f0a715c20304b74e0f 2024-12-11T22:41:03,632 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for 419f7368f61981f0a715c20304b74e0f 2024-12-11T22:41:03,632 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for 419f7368f61981f0a715c20304b74e0f 2024-12-11T22:41:03,633 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for de3803c768bc063bdc71e495af58eb48 2024-12-11T22:41:03,633 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for 419f7368f61981f0a715c20304b74e0f 2024-12-11T22:41:03,634 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened de3803c768bc063bdc71e495af58eb48; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60191882, jitterRate=-0.10307106375694275}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:41:03,634 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for de3803c768bc063bdc71e495af58eb48 2024-12-11T22:41:03,634 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for de3803c768bc063bdc71e495af58eb48: Running coprocessor pre-open hook at 1733956863625Writing region info on filesystem at 1733956863625Initializing all the Stores at 1733956863626 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733956863626Cleaning up temporary data from old regions at 1733956863631 (+5 ms)Running coprocessor post-open hooks at 1733956863634 (+3 ms)Region opened successfully at 1733956863634 2024-12-11T22:41:03,634 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened 419f7368f61981f0a715c20304b74e0f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65031990, jitterRate=-0.030947834253311157}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:41:03,634 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 419f7368f61981f0a715c20304b74e0f 2024-12-11T22:41:03,634 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for 419f7368f61981f0a715c20304b74e0f: Running coprocessor pre-open hook at 1733956863627Writing region info on filesystem at 1733956863627Initializing all the Stores at 1733956863628 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733956863628Cleaning up temporary data from old regions at 1733956863632 (+4 ms)Running coprocessor post-open hooks at 1733956863634 (+2 ms)Region opened successfully at 1733956863634 2024-12-11T22:41:03,635 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1733956859502.de3803c768bc063bdc71e495af58eb48., pid=79, masterSystemTime=1733956863585 2024-12-11T22:41:03,635 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1733956859502.419f7368f61981f0a715c20304b74e0f., pid=81, masterSystemTime=1733956863590 2024-12-11T22:41:03,637 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1733956859502.de3803c768bc063bdc71e495af58eb48. 2024-12-11T22:41:03,637 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened test,ggg,1733956859502.de3803c768bc063bdc71e495af58eb48. 2024-12-11T22:41:03,637 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(132): Open test,ppp,1733956859502.01df3c1c862de5abd063d235e117a115. 2024-12-11T22:41:03,637 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7752): Opening region: {ENCODED => 01df3c1c862de5abd063d235e117a115, NAME => 'test,ppp,1733956859502.01df3c1c862de5abd063d235e117a115.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-12-11T22:41:03,638 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 01df3c1c862de5abd063d235e117a115 2024-12-11T22:41:03,638 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(898): Instantiated test,ppp,1733956859502.01df3c1c862de5abd063d235e117a115.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:41:03,638 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7794): checking encryption for 01df3c1c862de5abd063d235e117a115 2024-12-11T22:41:03,638 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7797): checking classloading for 01df3c1c862de5abd063d235e117a115 2024-12-11T22:41:03,638 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1733956859502.419f7368f61981f0a715c20304b74e0f. 2024-12-11T22:41:03,639 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(153): Opened test,fff,1733956859502.419f7368f61981f0a715c20304b74e0f. 2024-12-11T22:41:03,639 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,rrr,1733956859502.f219e06826a4a3ede110adacdf28f6f4. 2024-12-11T22:41:03,639 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => f219e06826a4a3ede110adacdf28f6f4, NAME => 'test,rrr,1733956859502.f219e06826a4a3ede110adacdf28f6f4.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-12-11T22:41:03,639 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f219e06826a4a3ede110adacdf28f6f4 2024-12-11T22:41:03,639 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,rrr,1733956859502.f219e06826a4a3ede110adacdf28f6f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:41:03,639 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for f219e06826a4a3ede110adacdf28f6f4 2024-12-11T22:41:03,639 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for f219e06826a4a3ede110adacdf28f6f4 2024-12-11T22:41:03,640 INFO [StoreOpener-01df3c1c862de5abd063d235e117a115-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 01df3c1c862de5abd063d235e117a115 2024-12-11T22:41:03,641 INFO [StoreOpener-f219e06826a4a3ede110adacdf28f6f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f219e06826a4a3ede110adacdf28f6f4 2024-12-11T22:41:03,641 INFO [StoreOpener-01df3c1c862de5abd063d235e117a115-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 01df3c1c862de5abd063d235e117a115 columnFamilyName family 2024-12-11T22:41:03,642 DEBUG [StoreOpener-01df3c1c862de5abd063d235e117a115-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:41:03,642 INFO [StoreOpener-f219e06826a4a3ede110adacdf28f6f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f219e06826a4a3ede110adacdf28f6f4 columnFamilyName family 2024-12-11T22:41:03,642 DEBUG [StoreOpener-f219e06826a4a3ede110adacdf28f6f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:41:03,642 INFO [StoreOpener-01df3c1c862de5abd063d235e117a115-1 {}] regionserver.HStore(327): Store=01df3c1c862de5abd063d235e117a115/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:41:03,642 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1038): replaying wal for 01df3c1c862de5abd063d235e117a115 2024-12-11T22:41:03,642 INFO [StoreOpener-f219e06826a4a3ede110adacdf28f6f4-1 {}] regionserver.HStore(327): Store=f219e06826a4a3ede110adacdf28f6f4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:41:03,643 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for f219e06826a4a3ede110adacdf28f6f4 2024-12-11T22:41:03,643 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/01df3c1c862de5abd063d235e117a115 2024-12-11T22:41:03,644 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/f219e06826a4a3ede110adacdf28f6f4 2024-12-11T22:41:03,644 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/01df3c1c862de5abd063d235e117a115 2024-12-11T22:41:03,645 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1048): stopping wal replay for 01df3c1c862de5abd063d235e117a115 2024-12-11T22:41:03,645 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1060): Cleaning up temporary data for 01df3c1c862de5abd063d235e117a115 2024-12-11T22:41:03,645 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/f219e06826a4a3ede110adacdf28f6f4 2024-12-11T22:41:03,645 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for f219e06826a4a3ede110adacdf28f6f4 2024-12-11T22:41:03,646 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for f219e06826a4a3ede110adacdf28f6f4 2024-12-11T22:41:03,647 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for f219e06826a4a3ede110adacdf28f6f4 2024-12-11T22:41:03,647 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1093): writing seq id for 01df3c1c862de5abd063d235e117a115 2024-12-11T22:41:03,648 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened f219e06826a4a3ede110adacdf28f6f4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72797403, jitterRate=0.08476583659648895}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:41:03,648 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1114): Opened 01df3c1c862de5abd063d235e117a115; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65834986, jitterRate=-0.018982261419296265}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:41:03,648 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 01df3c1c862de5abd063d235e117a115 2024-12-11T22:41:03,648 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f219e06826a4a3ede110adacdf28f6f4 2024-12-11T22:41:03,648 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for f219e06826a4a3ede110adacdf28f6f4: Running coprocessor pre-open hook at 1733956863639Writing region info on filesystem at 1733956863639Initializing all the Stores at 1733956863641 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733956863641Cleaning up temporary data from old regions at 1733956863646 (+5 ms)Running coprocessor post-open hooks at 1733956863648 (+2 ms)Region opened successfully at 1733956863648 2024-12-11T22:41:03,648 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1006): Region open journal for 01df3c1c862de5abd063d235e117a115: Running coprocessor pre-open hook at 1733956863638Writing region info on filesystem at 1733956863638Initializing all the Stores at 1733956863639 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733956863640 (+1 ms)Cleaning up temporary data from old regions at 1733956863645 (+5 ms)Running coprocessor post-open hooks at 1733956863648 (+3 ms)Region opened successfully at 1733956863648 2024-12-11T22:41:03,649 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1733956859502.01df3c1c862de5abd063d235e117a115., pid=80, masterSystemTime=1733956863585 2024-12-11T22:41:03,649 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1733956859502.f219e06826a4a3ede110adacdf28f6f4., pid=76, masterSystemTime=1733956863590 2024-12-11T22:41:03,651 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1733956859502.01df3c1c862de5abd063d235e117a115. 2024-12-11T22:41:03,651 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(153): Opened test,ppp,1733956859502.01df3c1c862de5abd063d235e117a115. 2024-12-11T22:41:03,652 DEBUG [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1733956859502.f219e06826a4a3ede110adacdf28f6f4. 2024-12-11T22:41:03,652 INFO [RS_OPEN_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened test,rrr,1733956859502.f219e06826a4a3ede110adacdf28f6f4. 2024-12-11T22:41:03,660 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-11T22:41:03,660 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:03,660 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-11T22:41:03,660 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-11T22:41:03,661 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-11T22:41:03,661 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-11T22:41:03,661 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-11T22:41:03,661 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-11T22:41:03,662 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-11T22:41:03,667 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-11T22:41:03,667 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-11T22:41:03,667 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-11T22:41:03,667 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,667 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,668 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,668 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,668 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,668 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,668 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,668 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,668 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,668 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,668 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,668 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,668 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,668 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,668 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,668 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=82, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,668 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-11T22:41:03,668 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-11T22:41:03,668 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,668 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-11T22:41:03,670 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-11T22:41:03,670 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f666d29f964372d8719d169fcecef641, REOPEN/MOVE 2024-12-11T22:41:03,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true 2024-12-11T22:41:03,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-11T22:41:03,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:03,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN 2024-12-11T22:41:03,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN 2024-12-11T22:41:03,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN 2024-12-11T22:41:03,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN 2024-12-11T22:41:03,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN 2024-12-11T22:41:03,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN 2024-12-11T22:41:03,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN 2024-12-11T22:41:03,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN 2024-12-11T22:41:03,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure b30c1b01a8241f23347f03f93df7f090, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:03,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure f219e06826a4a3ede110adacdf28f6f4, server=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0e02a7a4f8d57169d1f435eabc00839, server=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,676 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:03,676 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure de3803c768bc063bdc71e495af58eb48, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:03,676 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 01df3c1c862de5abd063d235e117a115, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:03,676 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 419f7368f61981f0a715c20304b74e0f, server=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,676 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=82, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure ad4bb3db7f3f05f24b6115fc887d6ab5, server=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,676 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,676 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,676 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,676 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,676 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,34703,1733956857011 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,676 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,676 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,676 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,676 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN 2024-12-11T22:41:03,676 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,676 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN 2024-12-11T22:41:03,676 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,676 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN 2024-12-11T22:41:03,676 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,676 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN 2024-12-11T22:41:03,676 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,676 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN 2024-12-11T22:41:03,676 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,676 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN 2024-12-11T22:41:03,676 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,676 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN 2024-12-11T22:41:03,676 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,676 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN 2024-12-11T22:41:03,676 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure b30c1b01a8241f23347f03f93df7f090, server=56d7a8405f4a,34703,1733956857011 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,677 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure f219e06826a4a3ede110adacdf28f6f4, server=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,677 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0e02a7a4f8d57169d1f435eabc00839, server=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,677 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,677 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure de3803c768bc063bdc71e495af58eb48, server=56d7a8405f4a,34703,1733956857011 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,677 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 01df3c1c862de5abd063d235e117a115, server=56d7a8405f4a,34703,1733956857011 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,677 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 419f7368f61981f0a715c20304b74e0f, server=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,677 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=82, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure ad4bb3db7f3f05f24b6115fc887d6ab5, server=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,677 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 15 msec 2024-12-11T22:41:03,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=65 2024-12-11T22:41:03,679 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=56d7a8405f4a,34703,1733956857011, table=hbase:meta, region=1588230740 2024-12-11T22:41:03,679 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 56d7a8405f4a,34703,1733956857011, state=OPEN 2024-12-11T22:41:03,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:03,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:03,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:03,696 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:03,696 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:03,696 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:03,697 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN to state=OFFLINE, location=null, table=test, region=b30c1b01a8241f23347f03f93df7f090 to restore RIT 2024-12-11T22:41:03,697 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN to state=OFFLINE, location=null, table=test, region=f219e06826a4a3ede110adacdf28f6f4 to restore RIT 2024-12-11T22:41:03,697 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN to state=OFFLINE, location=null, table=test, region=5a93ec512c97cf46e8942b91f4c8f088 to restore RIT 2024-12-11T22:41:03,697 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN to state=OFFLINE, location=null, table=test, region=419f7368f61981f0a715c20304b74e0f to restore RIT 2024-12-11T22:41:03,697 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN to state=OFFLINE, location=null, table=test, region=de3803c768bc063bdc71e495af58eb48 to restore RIT 2024-12-11T22:41:03,697 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN to state=OFFLINE, location=null, table=test, region=a0e02a7a4f8d57169d1f435eabc00839 to restore RIT 2024-12-11T22:41:03,697 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN to state=OFFLINE, location=null, table=test, region=01df3c1c862de5abd063d235e117a115 to restore RIT 2024-12-11T22:41:03,697 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN to state=OFFLINE, location=null, table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5 to restore RIT 2024-12-11T22:41:03,698 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733956893698 2024-12-11T22:41:03,698 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-11T22:41:03,698 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:03,700 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-11T22:41:03,702 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-11T22:41:03,702 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 419f7368f61981f0a715c20304b74e0f, server=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:03,702 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 419f7368f61981f0a715c20304b74e0f, server=56d7a8405f4a,40421,1733956857131 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:03,724 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a93ec512c97cf46e8942b91f4c8f088, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,725 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66aac9e74e8263b0b8914fdc86eec66c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,725 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6801ac4c1cb1416e045d462c592d2dc5, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,725 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f169feb3190772cec56c0ae51d57c26, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,725 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9e94fba4d03035b5feaab607eb13853, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,726 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419f7368f61981f0a715c20304b74e0f, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,726 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de3803c768bc063bdc71e495af58eb48, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,726 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b31ee44b6609c355adc37986b13d239c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,726 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00fd1046c94e7f507c0ee43e6555d034, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,727 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad4bb3db7f3f05f24b6115fc887d6ab5, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,727 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6561ead2eaa9f60da54d4ccff5474a28, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,727 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445df7fd625f94e0eb0712fcd777ebda, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,727 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f666d29f964372d8719d169fcecef641, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=5 2024-12-11T22:41:03,727 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f30d8a32c00877e2cf6f22e7f50620, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,728 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1a38f11b48b53983535b863fd4392ff, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,728 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01df3c1c862de5abd063d235e117a115, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,728 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f551d316d038ca1e1403bd05f890572, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,728 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f219e06826a4a3ede110adacdf28f6f4, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=daa100e1546a41747fac83159daee190, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da518288db554a985096494f24adc59f, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7dbe11f45ae33a883424947e613c7c6, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b30c1b01a8241f23347f03f93df7f090, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860d604009b38831efaa4f23b2d17596, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0e02a7a4f8d57169d1f435eabc00839, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f19f540286846a30911c41a83b97457e, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff9e1faa3192d62c84407efca19e0a89, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,730 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-11T22:41:03,730 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733956923730 2024-12-11T22:41:03,730 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733956983730 2024-12-11T22:41:03,730 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 31 msec 2024-12-11T22:41:03,730 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-11T22:41:03,730 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:03,730 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-11T22:41:03,730 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-11T22:41:03,731 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-11T22:41:03,731 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-11T22:41:03,731 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-11T22:41:03,731 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-11T22:41:03,732 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-11T22:41:03,737 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-11T22:41:03,737 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-11T22:41:03,737 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-11T22:41:03,737 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,737 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,737 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,737 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,737 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,738 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,738 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,738 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,738 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,738 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,738 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,738 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,738 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,738 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,738 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,738 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-11T22:41:03,738 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-11T22:41:03,738 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,738 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,739 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-11T22:41:03,740 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-11T22:41:03,740 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f666d29f964372d8719d169fcecef641, REOPEN/MOVE 2024-12-11T22:41:03,744 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true 2024-12-11T22:41:03,744 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,744 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,744 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-11T22:41:03,744 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:03,744 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,744 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,744 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN 2024-12-11T22:41:03,744 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN 2024-12-11T22:41:03,744 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN 2024-12-11T22:41:03,744 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN 2024-12-11T22:41:03,744 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN 2024-12-11T22:41:03,744 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN 2024-12-11T22:41:03,744 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN 2024-12-11T22:41:03,744 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN 2024-12-11T22:41:03,744 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure b30c1b01a8241f23347f03f93df7f090, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:03,744 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure f219e06826a4a3ede110adacdf28f6f4, server=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,745 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0e02a7a4f8d57169d1f435eabc00839, server=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,745 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:03,745 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure de3803c768bc063bdc71e495af58eb48, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:03,745 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 01df3c1c862de5abd063d235e117a115, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:03,745 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 419f7368f61981f0a715c20304b74e0f, server=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,745 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,745 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,745 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,745 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,745 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,34703,1733956857011 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,745 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,745 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,745 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,745 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN 2024-12-11T22:41:03,745 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,745 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN 2024-12-11T22:41:03,745 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,746 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN 2024-12-11T22:41:03,746 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,746 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN 2024-12-11T22:41:03,746 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,746 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN 2024-12-11T22:41:03,746 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,746 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN 2024-12-11T22:41:03,746 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,746 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN 2024-12-11T22:41:03,746 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,746 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN 2024-12-11T22:41:03,746 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure b30c1b01a8241f23347f03f93df7f090, server=56d7a8405f4a,34703,1733956857011 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,746 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure f219e06826a4a3ede110adacdf28f6f4, server=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,746 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0e02a7a4f8d57169d1f435eabc00839, server=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,746 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,746 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure de3803c768bc063bdc71e495af58eb48, server=56d7a8405f4a,34703,1733956857011 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,746 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 01df3c1c862de5abd063d235e117a115, server=56d7a8405f4a,34703,1733956857011 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,746 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 419f7368f61981f0a715c20304b74e0f, server=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,746 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 14 msec 2024-12-11T22:41:03,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=65 2024-12-11T22:41:03,748 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=56d7a8405f4a,34703,1733956857011, table=hbase:meta, region=1588230740 2024-12-11T22:41:03,748 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 56d7a8405f4a,34703,1733956857011, state=OPEN 2024-12-11T22:41:03,758 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34843 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:03,761 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34843 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:03,779 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:03,779 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:03,779 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:03,779 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:03,779 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:03,779 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:03,780 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN to state=OFFLINE, location=null, table=test, region=b30c1b01a8241f23347f03f93df7f090 to restore RIT 2024-12-11T22:41:03,780 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN to state=OFFLINE, location=null, table=test, region=f219e06826a4a3ede110adacdf28f6f4 to restore RIT 2024-12-11T22:41:03,780 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN to state=OFFLINE, location=null, table=test, region=5a93ec512c97cf46e8942b91f4c8f088 to restore RIT 2024-12-11T22:41:03,780 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN to state=OFFLINE, location=null, table=test, region=419f7368f61981f0a715c20304b74e0f to restore RIT 2024-12-11T22:41:03,780 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN to state=OFFLINE, location=null, table=test, region=de3803c768bc063bdc71e495af58eb48 to restore RIT 2024-12-11T22:41:03,780 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN to state=OFFLINE, location=null, table=test, region=a0e02a7a4f8d57169d1f435eabc00839 to restore RIT 2024-12-11T22:41:03,780 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN to state=OFFLINE, location=null, table=test, region=01df3c1c862de5abd063d235e117a115 to restore RIT 2024-12-11T22:41:03,780 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN to state=OFFLINE, location=null, table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5 to restore RIT 2024-12-11T22:41:03,780 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733956893780 2024-12-11T22:41:03,781 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-11T22:41:03,781 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:03,782 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-11T22:41:03,784 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-11T22:41:03,784 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=80, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 01df3c1c862de5abd063d235e117a115, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:03,784 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:03,784 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=80, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 01df3c1c862de5abd063d235e117a115, server=56d7a8405f4a,34703,1733956857011 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:03,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a93ec512c97cf46e8942b91f4c8f088, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,791 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66aac9e74e8263b0b8914fdc86eec66c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,791 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6801ac4c1cb1416e045d462c592d2dc5, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,791 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f169feb3190772cec56c0ae51d57c26, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,791 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9e94fba4d03035b5feaab607eb13853, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,792 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419f7368f61981f0a715c20304b74e0f, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,792 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de3803c768bc063bdc71e495af58eb48, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,792 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b31ee44b6609c355adc37986b13d239c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,792 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00fd1046c94e7f507c0ee43e6555d034, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,792 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad4bb3db7f3f05f24b6115fc887d6ab5, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,793 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6561ead2eaa9f60da54d4ccff5474a28, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,793 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445df7fd625f94e0eb0712fcd777ebda, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,793 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f666d29f964372d8719d169fcecef641, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=5 2024-12-11T22:41:03,793 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f30d8a32c00877e2cf6f22e7f50620, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,793 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1a38f11b48b53983535b863fd4392ff, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,794 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01df3c1c862de5abd063d235e117a115, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,794 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f551d316d038ca1e1403bd05f890572, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,794 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f219e06826a4a3ede110adacdf28f6f4, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,794 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=daa100e1546a41747fac83159daee190, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,794 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da518288db554a985096494f24adc59f, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,795 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7dbe11f45ae33a883424947e613c7c6, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,795 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b30c1b01a8241f23347f03f93df7f090, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,795 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860d604009b38831efaa4f23b2d17596, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,795 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0e02a7a4f8d57169d1f435eabc00839, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,796 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f19f540286846a30911c41a83b97457e, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,796 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff9e1faa3192d62c84407efca19e0a89, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,796 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-11T22:41:03,796 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733956923796 2024-12-11T22:41:03,796 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733956983796 2024-12-11T22:41:03,796 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 15 msec 2024-12-11T22:41:03,796 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-11T22:41:03,796 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:03,796 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-11T22:41:03,796 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-11T22:41:03,797 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-11T22:41:03,797 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-11T22:41:03,797 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-11T22:41:03,797 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-11T22:41:03,798 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-11T22:41:03,803 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-11T22:41:03,803 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-11T22:41:03,803 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-11T22:41:03,803 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,803 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,803 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,803 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,803 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,803 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,803 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,803 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,804 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,804 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,804 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,804 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,804 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,804 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,804 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-11T22:41:03,804 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-11T22:41:03,804 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,804 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,804 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-11T22:41:03,805 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-11T22:41:03,806 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f666d29f964372d8719d169fcecef641, REOPEN/MOVE 2024-12-11T22:41:03,808 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true 2024-12-11T22:41:03,808 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,809 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,809 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-11T22:41:03,809 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:03,809 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,809 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,809 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN 2024-12-11T22:41:03,809 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN 2024-12-11T22:41:03,809 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN 2024-12-11T22:41:03,809 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN 2024-12-11T22:41:03,809 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN 2024-12-11T22:41:03,809 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN 2024-12-11T22:41:03,809 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN 2024-12-11T22:41:03,809 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN 2024-12-11T22:41:03,809 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure b30c1b01a8241f23347f03f93df7f090, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:03,809 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure f219e06826a4a3ede110adacdf28f6f4, server=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,809 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0e02a7a4f8d57169d1f435eabc00839, server=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,809 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:03,809 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure de3803c768bc063bdc71e495af58eb48, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:03,809 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 01df3c1c862de5abd063d235e117a115, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:03,810 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,810 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,810 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,810 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,810 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,34703,1733956857011 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,810 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,810 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,810 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,810 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN 2024-12-11T22:41:03,810 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,810 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN 2024-12-11T22:41:03,810 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,810 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN 2024-12-11T22:41:03,810 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,810 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN 2024-12-11T22:41:03,810 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,810 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN 2024-12-11T22:41:03,810 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,810 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN 2024-12-11T22:41:03,811 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,811 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN 2024-12-11T22:41:03,811 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,811 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN 2024-12-11T22:41:03,811 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure b30c1b01a8241f23347f03f93df7f090, server=56d7a8405f4a,34703,1733956857011 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,811 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure f219e06826a4a3ede110adacdf28f6f4, server=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,811 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0e02a7a4f8d57169d1f435eabc00839, server=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,811 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,811 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure de3803c768bc063bdc71e495af58eb48, server=56d7a8405f4a,34703,1733956857011 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,811 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 01df3c1c862de5abd063d235e117a115, server=56d7a8405f4a,34703,1733956857011 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,811 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 12 msec 2024-12-11T22:41:03,812 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=65 2024-12-11T22:41:03,812 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=56d7a8405f4a,34703,1733956857011, table=hbase:meta, region=1588230740 2024-12-11T22:41:03,813 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 56d7a8405f4a,34703,1733956857011, state=OPEN 2024-12-11T22:41:03,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:03,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:03,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:03,820 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:03,820 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:03,820 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:03,821 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN to state=OFFLINE, location=null, table=test, region=b30c1b01a8241f23347f03f93df7f090 to restore RIT 2024-12-11T22:41:03,821 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN to state=OFFLINE, location=null, table=test, region=f219e06826a4a3ede110adacdf28f6f4 to restore RIT 2024-12-11T22:41:03,821 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN to state=OFFLINE, location=null, table=test, region=5a93ec512c97cf46e8942b91f4c8f088 to restore RIT 2024-12-11T22:41:03,821 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN to state=OFFLINE, location=null, table=test, region=419f7368f61981f0a715c20304b74e0f to restore RIT 2024-12-11T22:41:03,821 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN to state=OFFLINE, location=null, table=test, region=de3803c768bc063bdc71e495af58eb48 to restore RIT 2024-12-11T22:41:03,821 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN to state=OFFLINE, location=null, table=test, region=a0e02a7a4f8d57169d1f435eabc00839 to restore RIT 2024-12-11T22:41:03,822 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN to state=OFFLINE, location=null, table=test, region=01df3c1c862de5abd063d235e117a115 to restore RIT 2024-12-11T22:41:03,822 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN to state=OFFLINE, location=null, table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5 to restore RIT 2024-12-11T22:41:03,822 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733956893822 2024-12-11T22:41:03,822 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-11T22:41:03,822 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:03,824 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-11T22:41:03,825 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-11T22:41:03,825 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure de3803c768bc063bdc71e495af58eb48, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:03,825 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:03,825 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure de3803c768bc063bdc71e495af58eb48, server=56d7a8405f4a,34703,1733956857011 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:03,831 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a93ec512c97cf46e8942b91f4c8f088, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,831 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66aac9e74e8263b0b8914fdc86eec66c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,831 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6801ac4c1cb1416e045d462c592d2dc5, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,832 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f169feb3190772cec56c0ae51d57c26, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,832 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9e94fba4d03035b5feaab607eb13853, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,832 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419f7368f61981f0a715c20304b74e0f, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,832 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de3803c768bc063bdc71e495af58eb48, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,833 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b31ee44b6609c355adc37986b13d239c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,833 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00fd1046c94e7f507c0ee43e6555d034, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,833 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad4bb3db7f3f05f24b6115fc887d6ab5, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,833 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6561ead2eaa9f60da54d4ccff5474a28, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,834 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445df7fd625f94e0eb0712fcd777ebda, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,834 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f666d29f964372d8719d169fcecef641, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=5 2024-12-11T22:41:03,834 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f30d8a32c00877e2cf6f22e7f50620, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,834 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1a38f11b48b53983535b863fd4392ff, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,835 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01df3c1c862de5abd063d235e117a115, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,835 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f551d316d038ca1e1403bd05f890572, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,835 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f219e06826a4a3ede110adacdf28f6f4, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,835 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=daa100e1546a41747fac83159daee190, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,836 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da518288db554a985096494f24adc59f, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,836 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7dbe11f45ae33a883424947e613c7c6, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,836 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b30c1b01a8241f23347f03f93df7f090, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,836 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860d604009b38831efaa4f23b2d17596, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,836 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0e02a7a4f8d57169d1f435eabc00839, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,837 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f19f540286846a30911c41a83b97457e, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,837 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff9e1faa3192d62c84407efca19e0a89, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,837 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-11T22:41:03,837 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733956923837 2024-12-11T22:41:03,837 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733956983837 2024-12-11T22:41:03,837 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 14 msec 2024-12-11T22:41:03,837 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-11T22:41:03,837 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:03,837 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-11T22:41:03,837 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-11T22:41:03,838 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-11T22:41:03,838 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-11T22:41:03,838 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-11T22:41:03,838 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-11T22:41:03,839 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-11T22:41:03,843 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-11T22:41:03,843 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-11T22:41:03,843 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-11T22:41:03,843 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,843 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,843 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,843 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,843 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,843 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,843 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,843 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,843 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,844 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,844 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,844 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,844 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,844 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-11T22:41:03,844 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-11T22:41:03,844 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,844 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,844 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-11T22:41:03,845 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-11T22:41:03,846 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f666d29f964372d8719d169fcecef641, REOPEN/MOVE 2024-12-11T22:41:03,849 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true 2024-12-11T22:41:03,849 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-11T22:41:03,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:03,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN 2024-12-11T22:41:03,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN 2024-12-11T22:41:03,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN 2024-12-11T22:41:03,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN 2024-12-11T22:41:03,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN 2024-12-11T22:41:03,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN 2024-12-11T22:41:03,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN 2024-12-11T22:41:03,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN 2024-12-11T22:41:03,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure b30c1b01a8241f23347f03f93df7f090, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:03,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure f219e06826a4a3ede110adacdf28f6f4, server=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0e02a7a4f8d57169d1f435eabc00839, server=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,851 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:03,851 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure de3803c768bc063bdc71e495af58eb48, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:03,851 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,851 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,851 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,851 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,851 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,34703,1733956857011 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,851 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,851 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,851 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,851 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN 2024-12-11T22:41:03,851 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,851 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN 2024-12-11T22:41:03,851 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,851 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN 2024-12-11T22:41:03,851 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,851 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN 2024-12-11T22:41:03,852 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,852 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN 2024-12-11T22:41:03,852 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,852 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN 2024-12-11T22:41:03,852 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,852 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN 2024-12-11T22:41:03,852 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,852 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN 2024-12-11T22:41:03,852 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure b30c1b01a8241f23347f03f93df7f090, server=56d7a8405f4a,34703,1733956857011 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,852 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure f219e06826a4a3ede110adacdf28f6f4, server=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,852 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0e02a7a4f8d57169d1f435eabc00839, server=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,852 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,852 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure de3803c768bc063bdc71e495af58eb48, server=56d7a8405f4a,34703,1733956857011 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,852 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 13 msec 2024-12-11T22:41:03,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=65 2024-12-11T22:41:03,854 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=56d7a8405f4a,34703,1733956857011, table=hbase:meta, region=1588230740 2024-12-11T22:41:03,854 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 56d7a8405f4a,34703,1733956857011, state=OPEN 2024-12-11T22:41:03,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:03,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:03,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:03,862 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:03,862 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:03,862 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:03,863 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN to state=OFFLINE, location=null, table=test, region=b30c1b01a8241f23347f03f93df7f090 to restore RIT 2024-12-11T22:41:03,863 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN to state=OFFLINE, location=null, table=test, region=f219e06826a4a3ede110adacdf28f6f4 to restore RIT 2024-12-11T22:41:03,863 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN to state=OFFLINE, location=null, table=test, region=5a93ec512c97cf46e8942b91f4c8f088 to restore RIT 2024-12-11T22:41:03,863 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN to state=OFFLINE, location=null, table=test, region=419f7368f61981f0a715c20304b74e0f to restore RIT 2024-12-11T22:41:03,863 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN to state=OFFLINE, location=null, table=test, region=de3803c768bc063bdc71e495af58eb48 to restore RIT 2024-12-11T22:41:03,863 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN to state=OFFLINE, location=null, table=test, region=a0e02a7a4f8d57169d1f435eabc00839 to restore RIT 2024-12-11T22:41:03,863 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN to state=OFFLINE, location=null, table=test, region=01df3c1c862de5abd063d235e117a115 to restore RIT 2024-12-11T22:41:03,863 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN to state=OFFLINE, location=null, table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5 to restore RIT 2024-12-11T22:41:03,864 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733956893864 2024-12-11T22:41:03,864 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-11T22:41:03,864 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:03,866 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-11T22:41:03,867 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-11T22:41:03,867 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:03,867 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:03,867 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:03,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a93ec512c97cf46e8942b91f4c8f088, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,871 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66aac9e74e8263b0b8914fdc86eec66c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,871 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6801ac4c1cb1416e045d462c592d2dc5, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,871 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f169feb3190772cec56c0ae51d57c26, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,871 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9e94fba4d03035b5feaab607eb13853, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,871 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419f7368f61981f0a715c20304b74e0f, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,871 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de3803c768bc063bdc71e495af58eb48, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b31ee44b6609c355adc37986b13d239c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00fd1046c94e7f507c0ee43e6555d034, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad4bb3db7f3f05f24b6115fc887d6ab5, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6561ead2eaa9f60da54d4ccff5474a28, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445df7fd625f94e0eb0712fcd777ebda, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f666d29f964372d8719d169fcecef641, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=5 2024-12-11T22:41:03,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f30d8a32c00877e2cf6f22e7f50620, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1a38f11b48b53983535b863fd4392ff, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01df3c1c862de5abd063d235e117a115, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f551d316d038ca1e1403bd05f890572, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f219e06826a4a3ede110adacdf28f6f4, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=daa100e1546a41747fac83159daee190, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da518288db554a985096494f24adc59f, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7dbe11f45ae33a883424947e613c7c6, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b30c1b01a8241f23347f03f93df7f090, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860d604009b38831efaa4f23b2d17596, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0e02a7a4f8d57169d1f435eabc00839, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,874 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f19f540286846a30911c41a83b97457e, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,874 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff9e1faa3192d62c84407efca19e0a89, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,874 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-11T22:41:03,874 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733956923874 2024-12-11T22:41:03,874 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733956983874 2024-12-11T22:41:03,874 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-12-11T22:41:03,874 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-11T22:41:03,874 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:03,874 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-11T22:41:03,874 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-11T22:41:03,874 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-11T22:41:03,874 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-11T22:41:03,874 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-11T22:41:03,875 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-11T22:41:03,875 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-11T22:41:03,879 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-11T22:41:03,879 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-11T22:41:03,879 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-11T22:41:03,879 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,880 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,880 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,880 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,880 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,880 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,880 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,880 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,880 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,880 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,880 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,880 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,880 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-11T22:41:03,880 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-11T22:41:03,880 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,880 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,881 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-11T22:41:03,881 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34843 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:03,881 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34843 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:03,882 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-11T22:41:03,882 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f666d29f964372d8719d169fcecef641, REOPEN/MOVE 2024-12-11T22:41:03,885 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true 2024-12-11T22:41:03,885 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,885 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,886 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-11T22:41:03,886 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:03,886 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,886 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,886 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN 2024-12-11T22:41:03,886 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN 2024-12-11T22:41:03,886 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN 2024-12-11T22:41:03,886 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN 2024-12-11T22:41:03,886 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN 2024-12-11T22:41:03,886 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN 2024-12-11T22:41:03,886 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN 2024-12-11T22:41:03,886 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN 2024-12-11T22:41:03,886 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure b30c1b01a8241f23347f03f93df7f090, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:03,886 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure f219e06826a4a3ede110adacdf28f6f4, server=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,886 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0e02a7a4f8d57169d1f435eabc00839, server=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,886 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:03,886 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,887 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,887 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,887 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,887 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,34703,1733956857011 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,887 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,887 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,887 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,887 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN 2024-12-11T22:41:03,887 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,887 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN 2024-12-11T22:41:03,887 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,887 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN 2024-12-11T22:41:03,887 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,887 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN 2024-12-11T22:41:03,887 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,887 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN 2024-12-11T22:41:03,888 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,888 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN 2024-12-11T22:41:03,888 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,888 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN 2024-12-11T22:41:03,888 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,888 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN 2024-12-11T22:41:03,888 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure b30c1b01a8241f23347f03f93df7f090, server=56d7a8405f4a,34703,1733956857011 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,888 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure f219e06826a4a3ede110adacdf28f6f4, server=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,888 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0e02a7a4f8d57169d1f435eabc00839, server=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,888 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,888 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 12 msec 2024-12-11T22:41:03,889 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=65 2024-12-11T22:41:03,889 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=56d7a8405f4a,34703,1733956857011, table=hbase:meta, region=1588230740 2024-12-11T22:41:03,890 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 56d7a8405f4a,34703,1733956857011, state=OPEN 2024-12-11T22:41:03,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:03,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:03,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:03,896 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:03,896 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:03,896 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:03,896 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN to state=OFFLINE, location=null, table=test, region=b30c1b01a8241f23347f03f93df7f090 to restore RIT 2024-12-11T22:41:03,896 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN to state=OFFLINE, location=null, table=test, region=f219e06826a4a3ede110adacdf28f6f4 to restore RIT 2024-12-11T22:41:03,896 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN to state=OFFLINE, location=null, table=test, region=5a93ec512c97cf46e8942b91f4c8f088 to restore RIT 2024-12-11T22:41:03,897 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN to state=OFFLINE, location=null, table=test, region=419f7368f61981f0a715c20304b74e0f to restore RIT 2024-12-11T22:41:03,897 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN to state=OFFLINE, location=null, table=test, region=de3803c768bc063bdc71e495af58eb48 to restore RIT 2024-12-11T22:41:03,897 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN to state=OFFLINE, location=null, table=test, region=a0e02a7a4f8d57169d1f435eabc00839 to restore RIT 2024-12-11T22:41:03,897 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN to state=OFFLINE, location=null, table=test, region=01df3c1c862de5abd063d235e117a115 to restore RIT 2024-12-11T22:41:03,897 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN to state=OFFLINE, location=null, table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5 to restore RIT 2024-12-11T22:41:03,897 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733956893897 2024-12-11T22:41:03,897 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-11T22:41:03,897 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:03,899 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-11T22:41:03,901 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-11T22:41:03,901 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0e02a7a4f8d57169d1f435eabc00839, server=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,901 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:03,901 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0e02a7a4f8d57169d1f435eabc00839, server=56d7a8405f4a,40421,1733956857131 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:03,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a93ec512c97cf46e8942b91f4c8f088, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66aac9e74e8263b0b8914fdc86eec66c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6801ac4c1cb1416e045d462c592d2dc5, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f169feb3190772cec56c0ae51d57c26, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9e94fba4d03035b5feaab607eb13853, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419f7368f61981f0a715c20304b74e0f, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de3803c768bc063bdc71e495af58eb48, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b31ee44b6609c355adc37986b13d239c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00fd1046c94e7f507c0ee43e6555d034, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad4bb3db7f3f05f24b6115fc887d6ab5, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6561ead2eaa9f60da54d4ccff5474a28, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445df7fd625f94e0eb0712fcd777ebda, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f666d29f964372d8719d169fcecef641, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=5 2024-12-11T22:41:03,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f30d8a32c00877e2cf6f22e7f50620, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,907 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1a38f11b48b53983535b863fd4392ff, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,907 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01df3c1c862de5abd063d235e117a115, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,907 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f551d316d038ca1e1403bd05f890572, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,907 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f219e06826a4a3ede110adacdf28f6f4, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,908 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=daa100e1546a41747fac83159daee190, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,908 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da518288db554a985096494f24adc59f, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,908 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7dbe11f45ae33a883424947e613c7c6, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,908 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b30c1b01a8241f23347f03f93df7f090, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,908 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860d604009b38831efaa4f23b2d17596, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,909 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0e02a7a4f8d57169d1f435eabc00839, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,909 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f19f540286846a30911c41a83b97457e, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,909 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff9e1faa3192d62c84407efca19e0a89, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,909 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-11T22:41:03,909 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733956923909 2024-12-11T22:41:03,909 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733956983909 2024-12-11T22:41:03,909 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-12-11T22:41:03,909 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-11T22:41:03,909 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:03,909 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-11T22:41:03,909 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-11T22:41:03,910 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-11T22:41:03,910 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-11T22:41:03,910 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-11T22:41:03,910 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-11T22:41:03,911 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-11T22:41:03,913 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-11T22:41:03,913 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-11T22:41:03,913 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-11T22:41:03,913 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,913 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,913 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,913 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,913 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,913 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,913 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,913 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,913 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,913 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,913 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,913 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-11T22:41:03,913 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-11T22:41:03,913 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,913 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,914 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-11T22:41:03,914 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-11T22:41:03,914 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f666d29f964372d8719d169fcecef641, REOPEN/MOVE 2024-12-11T22:41:03,915 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true 2024-12-11T22:41:03,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-11T22:41:03,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:03,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN 2024-12-11T22:41:03,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN 2024-12-11T22:41:03,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN 2024-12-11T22:41:03,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN 2024-12-11T22:41:03,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN 2024-12-11T22:41:03,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN 2024-12-11T22:41:03,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN 2024-12-11T22:41:03,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN 2024-12-11T22:41:03,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure b30c1b01a8241f23347f03f93df7f090, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:03,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure f219e06826a4a3ede110adacdf28f6f4, server=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0e02a7a4f8d57169d1f435eabc00839, server=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,916 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,916 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,916 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,916 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,916 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,34703,1733956857011 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,916 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,916 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,916 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,916 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN 2024-12-11T22:41:03,916 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,916 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN 2024-12-11T22:41:03,916 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,916 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN 2024-12-11T22:41:03,916 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,917 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN 2024-12-11T22:41:03,917 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,917 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN 2024-12-11T22:41:03,917 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,917 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN 2024-12-11T22:41:03,917 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,917 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN 2024-12-11T22:41:03,917 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,917 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN 2024-12-11T22:41:03,917 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure b30c1b01a8241f23347f03f93df7f090, server=56d7a8405f4a,34703,1733956857011 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,917 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure f219e06826a4a3ede110adacdf28f6f4, server=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,917 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0e02a7a4f8d57169d1f435eabc00839, server=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,917 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-12-11T22:41:03,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=65 2024-12-11T22:41:03,918 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=56d7a8405f4a,34703,1733956857011, table=hbase:meta, region=1588230740 2024-12-11T22:41:03,918 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 56d7a8405f4a,34703,1733956857011, state=OPEN 2024-12-11T22:41:03,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:03,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:03,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:03,929 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:03,929 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:03,929 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:03,929 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN to state=OFFLINE, location=null, table=test, region=b30c1b01a8241f23347f03f93df7f090 to restore RIT 2024-12-11T22:41:03,929 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN to state=OFFLINE, location=null, table=test, region=f219e06826a4a3ede110adacdf28f6f4 to restore RIT 2024-12-11T22:41:03,930 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN to state=OFFLINE, location=null, table=test, region=5a93ec512c97cf46e8942b91f4c8f088 to restore RIT 2024-12-11T22:41:03,930 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN to state=OFFLINE, location=null, table=test, region=419f7368f61981f0a715c20304b74e0f to restore RIT 2024-12-11T22:41:03,930 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN to state=OFFLINE, location=null, table=test, region=de3803c768bc063bdc71e495af58eb48 to restore RIT 2024-12-11T22:41:03,930 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN to state=OFFLINE, location=null, table=test, region=a0e02a7a4f8d57169d1f435eabc00839 to restore RIT 2024-12-11T22:41:03,930 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN to state=OFFLINE, location=null, table=test, region=01df3c1c862de5abd063d235e117a115 to restore RIT 2024-12-11T22:41:03,930 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN to state=OFFLINE, location=null, table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5 to restore RIT 2024-12-11T22:41:03,930 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733956893930 2024-12-11T22:41:03,930 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-11T22:41:03,930 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:03,932 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-11T22:41:03,933 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-11T22:41:03,933 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure f219e06826a4a3ede110adacdf28f6f4, server=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:03,933 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure f219e06826a4a3ede110adacdf28f6f4, server=56d7a8405f4a,40421,1733956857131 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:03,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a93ec512c97cf46e8942b91f4c8f088, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66aac9e74e8263b0b8914fdc86eec66c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6801ac4c1cb1416e045d462c592d2dc5, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f169feb3190772cec56c0ae51d57c26, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9e94fba4d03035b5feaab607eb13853, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419f7368f61981f0a715c20304b74e0f, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de3803c768bc063bdc71e495af58eb48, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b31ee44b6609c355adc37986b13d239c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00fd1046c94e7f507c0ee43e6555d034, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad4bb3db7f3f05f24b6115fc887d6ab5, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6561ead2eaa9f60da54d4ccff5474a28, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445df7fd625f94e0eb0712fcd777ebda, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f666d29f964372d8719d169fcecef641, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=5 2024-12-11T22:41:03,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f30d8a32c00877e2cf6f22e7f50620, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1a38f11b48b53983535b863fd4392ff, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01df3c1c862de5abd063d235e117a115, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f551d316d038ca1e1403bd05f890572, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f219e06826a4a3ede110adacdf28f6f4, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=daa100e1546a41747fac83159daee190, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da518288db554a985096494f24adc59f, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7dbe11f45ae33a883424947e613c7c6, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b30c1b01a8241f23347f03f93df7f090, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860d604009b38831efaa4f23b2d17596, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0e02a7a4f8d57169d1f435eabc00839, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f19f540286846a30911c41a83b97457e, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,939 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff9e1faa3192d62c84407efca19e0a89, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,939 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-11T22:41:03,939 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733956923939 2024-12-11T22:41:03,939 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733956983939 2024-12-11T22:41:03,939 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-12-11T22:41:03,939 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-11T22:41:03,939 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:03,939 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-11T22:41:03,939 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-11T22:41:03,939 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-11T22:41:03,939 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-11T22:41:03,939 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-11T22:41:03,940 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-11T22:41:03,940 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-11T22:41:03,943 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-11T22:41:03,943 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-11T22:41:03,943 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-11T22:41:03,943 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,943 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,943 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,943 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,944 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,944 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,944 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,944 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,944 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,944 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,944 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-11T22:41:03,944 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-11T22:41:03,944 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,944 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,944 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-11T22:41:03,945 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-11T22:41:03,945 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f666d29f964372d8719d169fcecef641, REOPEN/MOVE 2024-12-11T22:41:03,947 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true 2024-12-11T22:41:03,947 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,947 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,947 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-11T22:41:03,947 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:03,947 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,947 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,947 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN 2024-12-11T22:41:03,947 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN 2024-12-11T22:41:03,947 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN 2024-12-11T22:41:03,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN 2024-12-11T22:41:03,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN 2024-12-11T22:41:03,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN 2024-12-11T22:41:03,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN 2024-12-11T22:41:03,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN 2024-12-11T22:41:03,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure b30c1b01a8241f23347f03f93df7f090, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:03,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure f219e06826a4a3ede110adacdf28f6f4, server=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,948 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,948 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,948 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,948 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,948 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,34703,1733956857011 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,948 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,948 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,948 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,948 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN 2024-12-11T22:41:03,948 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,948 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN 2024-12-11T22:41:03,948 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,948 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN 2024-12-11T22:41:03,948 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,948 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN 2024-12-11T22:41:03,948 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,948 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN 2024-12-11T22:41:03,948 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,948 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN 2024-12-11T22:41:03,948 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,949 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN 2024-12-11T22:41:03,949 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,949 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN 2024-12-11T22:41:03,949 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure b30c1b01a8241f23347f03f93df7f090, server=56d7a8405f4a,34703,1733956857011 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,949 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure f219e06826a4a3ede110adacdf28f6f4, server=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,949 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-12-11T22:41:03,950 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=65 2024-12-11T22:41:03,950 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=56d7a8405f4a,34703,1733956857011, table=hbase:meta, region=1588230740 2024-12-11T22:41:03,950 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 56d7a8405f4a,34703,1733956857011, state=OPEN 2024-12-11T22:41:03,961 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:03,961 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:03,961 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:03,961 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:03,961 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:03,961 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:03,962 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN to state=OFFLINE, location=null, table=test, region=b30c1b01a8241f23347f03f93df7f090 to restore RIT 2024-12-11T22:41:03,962 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN to state=OFFLINE, location=null, table=test, region=f219e06826a4a3ede110adacdf28f6f4 to restore RIT 2024-12-11T22:41:03,962 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN to state=OFFLINE, location=null, table=test, region=5a93ec512c97cf46e8942b91f4c8f088 to restore RIT 2024-12-11T22:41:03,962 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN to state=OFFLINE, location=null, table=test, region=419f7368f61981f0a715c20304b74e0f to restore RIT 2024-12-11T22:41:03,962 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN to state=OFFLINE, location=null, table=test, region=de3803c768bc063bdc71e495af58eb48 to restore RIT 2024-12-11T22:41:03,962 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN to state=OFFLINE, location=null, table=test, region=a0e02a7a4f8d57169d1f435eabc00839 to restore RIT 2024-12-11T22:41:03,962 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN to state=OFFLINE, location=null, table=test, region=01df3c1c862de5abd063d235e117a115 to restore RIT 2024-12-11T22:41:03,963 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN to state=OFFLINE, location=null, table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5 to restore RIT 2024-12-11T22:41:03,963 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733956893963 2024-12-11T22:41:03,963 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-11T22:41:03,963 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:03,964 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-11T22:41:03,965 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-11T22:41:03,965 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure b30c1b01a8241f23347f03f93df7f090, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:03,965 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:03,965 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure b30c1b01a8241f23347f03f93df7f090, server=56d7a8405f4a,34703,1733956857011 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:03,969 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a93ec512c97cf46e8942b91f4c8f088, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,969 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66aac9e74e8263b0b8914fdc86eec66c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6801ac4c1cb1416e045d462c592d2dc5, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f169feb3190772cec56c0ae51d57c26, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9e94fba4d03035b5feaab607eb13853, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419f7368f61981f0a715c20304b74e0f, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de3803c768bc063bdc71e495af58eb48, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b31ee44b6609c355adc37986b13d239c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00fd1046c94e7f507c0ee43e6555d034, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,971 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad4bb3db7f3f05f24b6115fc887d6ab5, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,971 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6561ead2eaa9f60da54d4ccff5474a28, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,971 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445df7fd625f94e0eb0712fcd777ebda, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,971 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f666d29f964372d8719d169fcecef641, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=5 2024-12-11T22:41:03,971 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f30d8a32c00877e2cf6f22e7f50620, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,971 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1a38f11b48b53983535b863fd4392ff, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,972 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01df3c1c862de5abd063d235e117a115, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,972 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f551d316d038ca1e1403bd05f890572, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,972 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f219e06826a4a3ede110adacdf28f6f4, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,972 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=daa100e1546a41747fac83159daee190, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,972 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da518288db554a985096494f24adc59f, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,972 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7dbe11f45ae33a883424947e613c7c6, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,972 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b30c1b01a8241f23347f03f93df7f090, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,973 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860d604009b38831efaa4f23b2d17596, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,973 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0e02a7a4f8d57169d1f435eabc00839, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,973 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f19f540286846a30911c41a83b97457e, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:03,973 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff9e1faa3192d62c84407efca19e0a89, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:03,973 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-11T22:41:03,973 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733956923973 2024-12-11T22:41:03,973 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733956983973 2024-12-11T22:41:03,973 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-12-11T22:41:03,973 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-11T22:41:03,973 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:03,974 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-11T22:41:03,974 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-11T22:41:03,974 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-11T22:41:03,974 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-11T22:41:03,974 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-11T22:41:03,974 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-11T22:41:03,975 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-11T22:41:03,977 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-11T22:41:03,977 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-11T22:41:03,978 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-11T22:41:03,978 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,978 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,978 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,978 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,978 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,978 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,978 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,978 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,978 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,978 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-11T22:41:03,978 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-11T22:41:03,978 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:03,978 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:03,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-11T22:41:03,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-11T22:41:03,980 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f666d29f964372d8719d169fcecef641, REOPEN/MOVE 2024-12-11T22:41:03,982 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true 2024-12-11T22:41:03,982 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,982 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,982 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-11T22:41:03,982 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:03,982 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,982 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:03,982 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN 2024-12-11T22:41:03,983 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN 2024-12-11T22:41:03,983 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN 2024-12-11T22:41:03,983 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN 2024-12-11T22:41:03,983 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN 2024-12-11T22:41:03,983 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN 2024-12-11T22:41:03,983 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN 2024-12-11T22:41:03,983 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN 2024-12-11T22:41:03,983 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure b30c1b01a8241f23347f03f93df7f090, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:03,983 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,983 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,983 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,983 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,983 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,34703,1733956857011 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,983 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,983 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,983 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,983 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN 2024-12-11T22:41:03,983 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,984 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN 2024-12-11T22:41:03,984 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,984 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN 2024-12-11T22:41:03,984 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,984 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN 2024-12-11T22:41:03,984 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,984 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN 2024-12-11T22:41:03,984 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,984 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN 2024-12-11T22:41:03,984 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,984 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN 2024-12-11T22:41:03,984 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:03,984 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN 2024-12-11T22:41:03,984 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure b30c1b01a8241f23347f03f93df7f090, server=56d7a8405f4a,34703,1733956857011 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:03,984 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-12-11T22:41:03,985 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34843 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:03,985 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34843 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:03,985 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=65 2024-12-11T22:41:03,985 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=56d7a8405f4a,34703,1733956857011, table=hbase:meta, region=1588230740 2024-12-11T22:41:03,985 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 56d7a8405f4a,34703,1733956857011, state=OPEN 2024-12-11T22:41:03,995 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:03,995 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:03,995 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:03,995 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:03,995 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:03,995 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:03,996 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN to state=OFFLINE, location=null, table=test, region=b30c1b01a8241f23347f03f93df7f090 to restore RIT 2024-12-11T22:41:03,996 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN to state=OFFLINE, location=null, table=test, region=f219e06826a4a3ede110adacdf28f6f4 to restore RIT 2024-12-11T22:41:03,996 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN to state=OFFLINE, location=null, table=test, region=5a93ec512c97cf46e8942b91f4c8f088 to restore RIT 2024-12-11T22:41:03,996 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN to state=OFFLINE, location=null, table=test, region=419f7368f61981f0a715c20304b74e0f to restore RIT 2024-12-11T22:41:03,996 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN to state=OFFLINE, location=null, table=test, region=de3803c768bc063bdc71e495af58eb48 to restore RIT 2024-12-11T22:41:03,996 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN to state=OFFLINE, location=null, table=test, region=a0e02a7a4f8d57169d1f435eabc00839 to restore RIT 2024-12-11T22:41:03,996 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN to state=OFFLINE, location=null, table=test, region=01df3c1c862de5abd063d235e117a115 to restore RIT 2024-12-11T22:41:03,997 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN to state=OFFLINE, location=null, table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5 to restore RIT 2024-12-11T22:41:03,997 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733956893997 2024-12-11T22:41:03,997 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-11T22:41:03,997 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:03,998 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-11T22:41:03,999 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-11T22:41:03,999 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN 2024-12-11T22:41:03,999 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:04,000 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:04,004 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a93ec512c97cf46e8942b91f4c8f088, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,004 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66aac9e74e8263b0b8914fdc86eec66c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,004 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6801ac4c1cb1416e045d462c592d2dc5, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,004 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f169feb3190772cec56c0ae51d57c26, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,004 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9e94fba4d03035b5feaab607eb13853, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,005 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419f7368f61981f0a715c20304b74e0f, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,005 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de3803c768bc063bdc71e495af58eb48, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,005 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b31ee44b6609c355adc37986b13d239c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,005 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00fd1046c94e7f507c0ee43e6555d034, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,005 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad4bb3db7f3f05f24b6115fc887d6ab5, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,005 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6561ead2eaa9f60da54d4ccff5474a28, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,005 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445df7fd625f94e0eb0712fcd777ebda, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,006 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f666d29f964372d8719d169fcecef641, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=5 2024-12-11T22:41:04,006 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f30d8a32c00877e2cf6f22e7f50620, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,006 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1a38f11b48b53983535b863fd4392ff, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,006 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01df3c1c862de5abd063d235e117a115, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,006 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f551d316d038ca1e1403bd05f890572, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,006 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f219e06826a4a3ede110adacdf28f6f4, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,007 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=daa100e1546a41747fac83159daee190, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,007 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da518288db554a985096494f24adc59f, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,007 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7dbe11f45ae33a883424947e613c7c6, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,007 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b30c1b01a8241f23347f03f93df7f090, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,007 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860d604009b38831efaa4f23b2d17596, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,007 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0e02a7a4f8d57169d1f435eabc00839, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,008 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f19f540286846a30911c41a83b97457e, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,008 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff9e1faa3192d62c84407efca19e0a89, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,008 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-11T22:41:04,008 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733956924008 2024-12-11T22:41:04,008 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733956984008 2024-12-11T22:41:04,008 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-12-11T22:41:04,008 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-11T22:41:04,008 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:04,008 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-11T22:41:04,008 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-11T22:41:04,009 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-11T22:41:04,009 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-11T22:41:04,009 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-11T22:41:04,009 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-11T22:41:04,010 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-11T22:41:04,012 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-11T22:41:04,012 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-11T22:41:04,012 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-11T22:41:04,012 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,012 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,012 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,012 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,012 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,012 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,012 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,012 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,013 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-11T22:41:04,013 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-11T22:41:04,013 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,013 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:04,013 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-11T22:41:04,014 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-11T22:41:04,014 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f666d29f964372d8719d169fcecef641, REOPEN/MOVE 2024-12-11T22:41:04,015 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true 2024-12-11T22:41:04,015 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,015 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,015 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-11T22:41:04,016 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:04,016 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,016 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,016 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN 2024-12-11T22:41:04,016 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN 2024-12-11T22:41:04,016 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN 2024-12-11T22:41:04,016 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN 2024-12-11T22:41:04,016 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN 2024-12-11T22:41:04,016 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN 2024-12-11T22:41:04,016 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN 2024-12-11T22:41:04,016 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN 2024-12-11T22:41:04,016 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,016 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,016 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,016 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,016 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,34703,1733956857011 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,016 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,016 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,016 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,016 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN 2024-12-11T22:41:04,016 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,016 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN 2024-12-11T22:41:04,017 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,017 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN 2024-12-11T22:41:04,017 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,017 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN 2024-12-11T22:41:04,017 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,017 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN 2024-12-11T22:41:04,017 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,017 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN 2024-12-11T22:41:04,017 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,017 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN 2024-12-11T22:41:04,017 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,017 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN 2024-12-11T22:41:04,017 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-12-11T22:41:04,018 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=65 2024-12-11T22:41:04,018 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=56d7a8405f4a,34703,1733956857011, table=hbase:meta, region=1588230740 2024-12-11T22:41:04,019 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 56d7a8405f4a,34703,1733956857011, state=OPEN 2024-12-11T22:41:04,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,029 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,029 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,029 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,029 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN to state=OFFLINE, location=null, table=test, region=b30c1b01a8241f23347f03f93df7f090 to restore RIT 2024-12-11T22:41:04,030 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN to state=OFFLINE, location=null, table=test, region=f219e06826a4a3ede110adacdf28f6f4 to restore RIT 2024-12-11T22:41:04,030 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN to state=OFFLINE, location=null, table=test, region=5a93ec512c97cf46e8942b91f4c8f088 to restore RIT 2024-12-11T22:41:04,030 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN to state=OFFLINE, location=null, table=test, region=419f7368f61981f0a715c20304b74e0f to restore RIT 2024-12-11T22:41:04,030 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN to state=OFFLINE, location=null, table=test, region=de3803c768bc063bdc71e495af58eb48 to restore RIT 2024-12-11T22:41:04,030 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN to state=OFFLINE, location=null, table=test, region=a0e02a7a4f8d57169d1f435eabc00839 to restore RIT 2024-12-11T22:41:04,030 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN to state=OFFLINE, location=null, table=test, region=01df3c1c862de5abd063d235e117a115 to restore RIT 2024-12-11T22:41:04,030 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5, ASSIGN to state=OFFLINE, location=null, table=test, region=ad4bb3db7f3f05f24b6115fc887d6ab5 to restore RIT 2024-12-11T22:41:04,030 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733956894030 2024-12-11T22:41:04,030 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-11T22:41:04,030 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:04,032 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-11T22:41:04,033 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-11T22:41:04,033 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN 2024-12-11T22:41:04,033 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:04,033 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:04,036 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a93ec512c97cf46e8942b91f4c8f088, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,037 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66aac9e74e8263b0b8914fdc86eec66c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,037 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6801ac4c1cb1416e045d462c592d2dc5, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,037 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f169feb3190772cec56c0ae51d57c26, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,037 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9e94fba4d03035b5feaab607eb13853, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,037 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419f7368f61981f0a715c20304b74e0f, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,037 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de3803c768bc063bdc71e495af58eb48, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,038 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b31ee44b6609c355adc37986b13d239c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,038 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00fd1046c94e7f507c0ee43e6555d034, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,038 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad4bb3db7f3f05f24b6115fc887d6ab5, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,038 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6561ead2eaa9f60da54d4ccff5474a28, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,038 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445df7fd625f94e0eb0712fcd777ebda, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,038 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f666d29f964372d8719d169fcecef641, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=5 2024-12-11T22:41:04,038 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f30d8a32c00877e2cf6f22e7f50620, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,039 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1a38f11b48b53983535b863fd4392ff, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,039 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01df3c1c862de5abd063d235e117a115, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,039 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f551d316d038ca1e1403bd05f890572, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,039 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f219e06826a4a3ede110adacdf28f6f4, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,039 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=daa100e1546a41747fac83159daee190, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,039 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da518288db554a985096494f24adc59f, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,039 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7dbe11f45ae33a883424947e613c7c6, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,040 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b30c1b01a8241f23347f03f93df7f090, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,040 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860d604009b38831efaa4f23b2d17596, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,040 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0e02a7a4f8d57169d1f435eabc00839, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,040 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f19f540286846a30911c41a83b97457e, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,040 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff9e1faa3192d62c84407efca19e0a89, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,040 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-11T22:41:04,040 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733956924040 2024-12-11T22:41:04,040 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733956984040 2024-12-11T22:41:04,040 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-12-11T22:41:04,040 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-11T22:41:04,041 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:04,041 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-11T22:41:04,041 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-11T22:41:04,041 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-11T22:41:04,041 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-11T22:41:04,041 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-11T22:41:04,041 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-11T22:41:04,042 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-11T22:41:04,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-11T22:41:04,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-11T22:41:04,045 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-11T22:41:04,045 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,045 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,045 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,045 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,045 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,045 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,045 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,045 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-11T22:41:04,045 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-11T22:41:04,045 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,045 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:04,045 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-11T22:41:04,046 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-11T22:41:04,046 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f666d29f964372d8719d169fcecef641, REOPEN/MOVE 2024-12-11T22:41:04,047 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true 2024-12-11T22:41:04,047 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,047 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,047 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-11T22:41:04,047 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:04,048 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,048 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,048 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN 2024-12-11T22:41:04,048 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN 2024-12-11T22:41:04,048 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN 2024-12-11T22:41:04,048 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN 2024-12-11T22:41:04,048 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN 2024-12-11T22:41:04,048 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN 2024-12-11T22:41:04,048 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN 2024-12-11T22:41:04,048 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,048 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,048 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,048 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,048 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,34703,1733956857011 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,048 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,048 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,048 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,048 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN 2024-12-11T22:41:04,048 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,049 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN 2024-12-11T22:41:04,049 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,049 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN 2024-12-11T22:41:04,049 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,049 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN 2024-12-11T22:41:04,049 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,049 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN 2024-12-11T22:41:04,049 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,049 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN 2024-12-11T22:41:04,049 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,049 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN 2024-12-11T22:41:04,049 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-12-11T22:41:04,050 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=65 2024-12-11T22:41:04,050 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=56d7a8405f4a,34703,1733956857011, table=hbase:meta, region=1588230740 2024-12-11T22:41:04,050 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 56d7a8405f4a,34703,1733956857011, state=OPEN 2024-12-11T22:41:04,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,062 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,062 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,062 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,063 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN to state=OFFLINE, location=null, table=test, region=b30c1b01a8241f23347f03f93df7f090 to restore RIT 2024-12-11T22:41:04,063 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN to state=OFFLINE, location=null, table=test, region=f219e06826a4a3ede110adacdf28f6f4 to restore RIT 2024-12-11T22:41:04,063 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN to state=OFFLINE, location=null, table=test, region=5a93ec512c97cf46e8942b91f4c8f088 to restore RIT 2024-12-11T22:41:04,063 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN to state=OFFLINE, location=null, table=test, region=419f7368f61981f0a715c20304b74e0f to restore RIT 2024-12-11T22:41:04,063 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN to state=OFFLINE, location=null, table=test, region=de3803c768bc063bdc71e495af58eb48 to restore RIT 2024-12-11T22:41:04,063 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN to state=OFFLINE, location=null, table=test, region=a0e02a7a4f8d57169d1f435eabc00839 to restore RIT 2024-12-11T22:41:04,063 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01df3c1c862de5abd063d235e117a115, ASSIGN to state=OFFLINE, location=null, table=test, region=01df3c1c862de5abd063d235e117a115 to restore RIT 2024-12-11T22:41:04,064 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733956894064 2024-12-11T22:41:04,064 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-11T22:41:04,064 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:04,065 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-11T22:41:04,067 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-11T22:41:04,067 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN 2024-12-11T22:41:04,067 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:04,067 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:04,069 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a93ec512c97cf46e8942b91f4c8f088, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,069 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66aac9e74e8263b0b8914fdc86eec66c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,069 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6801ac4c1cb1416e045d462c592d2dc5, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,069 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f169feb3190772cec56c0ae51d57c26, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,070 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9e94fba4d03035b5feaab607eb13853, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,070 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419f7368f61981f0a715c20304b74e0f, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,070 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de3803c768bc063bdc71e495af58eb48, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,070 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b31ee44b6609c355adc37986b13d239c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,070 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00fd1046c94e7f507c0ee43e6555d034, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,070 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad4bb3db7f3f05f24b6115fc887d6ab5, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,070 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6561ead2eaa9f60da54d4ccff5474a28, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,070 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445df7fd625f94e0eb0712fcd777ebda, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,071 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f666d29f964372d8719d169fcecef641, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=5 2024-12-11T22:41:04,071 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f30d8a32c00877e2cf6f22e7f50620, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,071 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1a38f11b48b53983535b863fd4392ff, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,071 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01df3c1c862de5abd063d235e117a115, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,071 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f551d316d038ca1e1403bd05f890572, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,071 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f219e06826a4a3ede110adacdf28f6f4, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,072 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=daa100e1546a41747fac83159daee190, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,072 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da518288db554a985096494f24adc59f, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,072 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7dbe11f45ae33a883424947e613c7c6, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,072 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b30c1b01a8241f23347f03f93df7f090, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,072 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860d604009b38831efaa4f23b2d17596, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,072 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0e02a7a4f8d57169d1f435eabc00839, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,073 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f19f540286846a30911c41a83b97457e, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,073 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff9e1faa3192d62c84407efca19e0a89, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,073 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-11T22:41:04,073 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733956924073 2024-12-11T22:41:04,073 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733956984073 2024-12-11T22:41:04,073 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-12-11T22:41:04,073 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-11T22:41:04,073 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:04,073 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-11T22:41:04,073 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-11T22:41:04,073 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-11T22:41:04,074 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-11T22:41:04,074 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-11T22:41:04,074 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-11T22:41:04,075 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-11T22:41:04,077 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-11T22:41:04,077 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-11T22:41:04,077 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-11T22:41:04,077 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,077 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,077 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,077 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,077 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,077 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,077 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-11T22:41:04,077 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-11T22:41:04,077 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,077 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:04,078 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-11T22:41:04,078 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-11T22:41:04,078 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f666d29f964372d8719d169fcecef641, REOPEN/MOVE 2024-12-11T22:41:04,080 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true 2024-12-11T22:41:04,080 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,080 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,080 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-11T22:41:04,080 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:04,080 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,080 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,080 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN 2024-12-11T22:41:04,080 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN 2024-12-11T22:41:04,080 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN 2024-12-11T22:41:04,080 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN 2024-12-11T22:41:04,081 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN 2024-12-11T22:41:04,081 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN 2024-12-11T22:41:04,081 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,081 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,081 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,081 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,081 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,34703,1733956857011 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,081 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,081 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,081 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,081 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN 2024-12-11T22:41:04,081 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,081 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN 2024-12-11T22:41:04,081 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,081 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN 2024-12-11T22:41:04,081 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,081 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN 2024-12-11T22:41:04,081 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,081 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN 2024-12-11T22:41:04,081 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,081 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN 2024-12-11T22:41:04,082 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-12-11T22:41:04,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=65 2024-12-11T22:41:04,083 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=56d7a8405f4a,34703,1733956857011, table=hbase:meta, region=1588230740 2024-12-11T22:41:04,083 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 56d7a8405f4a,34703,1733956857011, state=OPEN 2024-12-11T22:41:04,088 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34843 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:04,088 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34843 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:04,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,095 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,095 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,095 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,095 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN to state=OFFLINE, location=null, table=test, region=b30c1b01a8241f23347f03f93df7f090 to restore RIT 2024-12-11T22:41:04,095 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN to state=OFFLINE, location=null, table=test, region=f219e06826a4a3ede110adacdf28f6f4 to restore RIT 2024-12-11T22:41:04,095 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN to state=OFFLINE, location=null, table=test, region=5a93ec512c97cf46e8942b91f4c8f088 to restore RIT 2024-12-11T22:41:04,096 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN to state=OFFLINE, location=null, table=test, region=419f7368f61981f0a715c20304b74e0f to restore RIT 2024-12-11T22:41:04,096 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN to state=OFFLINE, location=null, table=test, region=de3803c768bc063bdc71e495af58eb48 to restore RIT 2024-12-11T22:41:04,096 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0e02a7a4f8d57169d1f435eabc00839, ASSIGN to state=OFFLINE, location=null, table=test, region=a0e02a7a4f8d57169d1f435eabc00839 to restore RIT 2024-12-11T22:41:04,096 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733956894096 2024-12-11T22:41:04,096 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-11T22:41:04,096 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:04,097 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-11T22:41:04,098 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-11T22:41:04,098 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN 2024-12-11T22:41:04,098 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:04,098 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:04,100 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a93ec512c97cf46e8942b91f4c8f088, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,100 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66aac9e74e8263b0b8914fdc86eec66c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,101 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6801ac4c1cb1416e045d462c592d2dc5, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,101 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f169feb3190772cec56c0ae51d57c26, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,101 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9e94fba4d03035b5feaab607eb13853, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,101 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419f7368f61981f0a715c20304b74e0f, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,101 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de3803c768bc063bdc71e495af58eb48, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,101 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b31ee44b6609c355adc37986b13d239c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,101 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00fd1046c94e7f507c0ee43e6555d034, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,101 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad4bb3db7f3f05f24b6115fc887d6ab5, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,101 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6561ead2eaa9f60da54d4ccff5474a28, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,101 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445df7fd625f94e0eb0712fcd777ebda, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,102 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f666d29f964372d8719d169fcecef641, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=5 2024-12-11T22:41:04,102 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f30d8a32c00877e2cf6f22e7f50620, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,102 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1a38f11b48b53983535b863fd4392ff, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,102 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01df3c1c862de5abd063d235e117a115, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,102 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f551d316d038ca1e1403bd05f890572, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,102 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f219e06826a4a3ede110adacdf28f6f4, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,102 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=daa100e1546a41747fac83159daee190, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,102 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da518288db554a985096494f24adc59f, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,102 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7dbe11f45ae33a883424947e613c7c6, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,102 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b30c1b01a8241f23347f03f93df7f090, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,102 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860d604009b38831efaa4f23b2d17596, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,102 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0e02a7a4f8d57169d1f435eabc00839, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,103 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f19f540286846a30911c41a83b97457e, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,103 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff9e1faa3192d62c84407efca19e0a89, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,103 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-11T22:41:04,103 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733956924103 2024-12-11T22:41:04,103 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733956984103 2024-12-11T22:41:04,103 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-12-11T22:41:04,103 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-11T22:41:04,103 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:04,103 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-11T22:41:04,103 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-11T22:41:04,103 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-11T22:41:04,103 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-11T22:41:04,103 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-11T22:41:04,104 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-11T22:41:04,104 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-11T22:41:04,106 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-11T22:41:04,106 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-11T22:41:04,106 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-11T22:41:04,106 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,106 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,106 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,106 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,106 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,106 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-11T22:41:04,106 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-11T22:41:04,106 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,106 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:04,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-11T22:41:04,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-11T22:41:04,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f666d29f964372d8719d169fcecef641, REOPEN/MOVE 2024-12-11T22:41:04,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true 2024-12-11T22:41:04,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-11T22:41:04,108 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:04,108 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,108 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,108 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN 2024-12-11T22:41:04,108 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN 2024-12-11T22:41:04,108 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN 2024-12-11T22:41:04,108 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN 2024-12-11T22:41:04,108 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN 2024-12-11T22:41:04,108 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,108 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,108 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,108 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,108 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,34703,1733956857011 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,108 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,108 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,108 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,108 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN 2024-12-11T22:41:04,108 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,108 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN 2024-12-11T22:41:04,108 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,108 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN 2024-12-11T22:41:04,108 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,108 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN 2024-12-11T22:41:04,108 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,108 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN 2024-12-11T22:41:04,108 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-11T22:41:04,109 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=65 2024-12-11T22:41:04,109 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=56d7a8405f4a,34703,1733956857011, table=hbase:meta, region=1588230740 2024-12-11T22:41:04,109 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 56d7a8405f4a,34703,1733956857011, state=OPEN 2024-12-11T22:41:04,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,120 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,120 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,120 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,121 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN to state=OFFLINE, location=null, table=test, region=b30c1b01a8241f23347f03f93df7f090 to restore RIT 2024-12-11T22:41:04,121 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN to state=OFFLINE, location=null, table=test, region=f219e06826a4a3ede110adacdf28f6f4 to restore RIT 2024-12-11T22:41:04,121 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN to state=OFFLINE, location=null, table=test, region=5a93ec512c97cf46e8942b91f4c8f088 to restore RIT 2024-12-11T22:41:04,121 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN to state=OFFLINE, location=null, table=test, region=419f7368f61981f0a715c20304b74e0f to restore RIT 2024-12-11T22:41:04,121 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=de3803c768bc063bdc71e495af58eb48, ASSIGN to state=OFFLINE, location=null, table=test, region=de3803c768bc063bdc71e495af58eb48 to restore RIT 2024-12-11T22:41:04,121 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733956894121 2024-12-11T22:41:04,122 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-11T22:41:04,122 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:04,122 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-11T22:41:04,123 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-11T22:41:04,124 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN 2024-12-11T22:41:04,124 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:04,124 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:04,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a93ec512c97cf46e8942b91f4c8f088, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66aac9e74e8263b0b8914fdc86eec66c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6801ac4c1cb1416e045d462c592d2dc5, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f169feb3190772cec56c0ae51d57c26, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9e94fba4d03035b5feaab607eb13853, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419f7368f61981f0a715c20304b74e0f, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de3803c768bc063bdc71e495af58eb48, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b31ee44b6609c355adc37986b13d239c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00fd1046c94e7f507c0ee43e6555d034, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad4bb3db7f3f05f24b6115fc887d6ab5, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,128 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6561ead2eaa9f60da54d4ccff5474a28, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,128 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445df7fd625f94e0eb0712fcd777ebda, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,128 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f666d29f964372d8719d169fcecef641, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=5 2024-12-11T22:41:04,128 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f30d8a32c00877e2cf6f22e7f50620, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,128 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1a38f11b48b53983535b863fd4392ff, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,128 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01df3c1c862de5abd063d235e117a115, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,128 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f551d316d038ca1e1403bd05f890572, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,128 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f219e06826a4a3ede110adacdf28f6f4, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,129 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=daa100e1546a41747fac83159daee190, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,129 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da518288db554a985096494f24adc59f, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,129 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7dbe11f45ae33a883424947e613c7c6, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,129 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b30c1b01a8241f23347f03f93df7f090, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,129 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860d604009b38831efaa4f23b2d17596, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,129 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0e02a7a4f8d57169d1f435eabc00839, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,129 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f19f540286846a30911c41a83b97457e, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,129 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff9e1faa3192d62c84407efca19e0a89, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,130 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-11T22:41:04,130 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733956924130 2024-12-11T22:41:04,130 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733956984130 2024-12-11T22:41:04,130 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-12-11T22:41:04,130 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-11T22:41:04,130 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:04,130 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-11T22:41:04,130 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-11T22:41:04,130 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-11T22:41:04,130 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-11T22:41:04,130 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-11T22:41:04,131 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-11T22:41:04,132 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-11T22:41:04,133 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-11T22:41:04,133 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-11T22:41:04,134 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-11T22:41:04,134 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,134 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,134 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,134 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,134 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-11T22:41:04,134 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-11T22:41:04,134 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,134 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:04,134 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-11T22:41:04,135 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-11T22:41:04,135 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f666d29f964372d8719d169fcecef641, REOPEN/MOVE 2024-12-11T22:41:04,136 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true 2024-12-11T22:41:04,136 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,136 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,136 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-11T22:41:04,136 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:04,136 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,136 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,136 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN 2024-12-11T22:41:04,136 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN 2024-12-11T22:41:04,136 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN 2024-12-11T22:41:04,136 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN 2024-12-11T22:41:04,136 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,136 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,136 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,136 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,136 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,34703,1733956857011 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,136 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,137 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,137 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,137 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN 2024-12-11T22:41:04,137 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,137 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN 2024-12-11T22:41:04,137 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,137 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN 2024-12-11T22:41:04,137 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,137 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN 2024-12-11T22:41:04,137 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-11T22:41:04,138 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=65 2024-12-11T22:41:04,138 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=56d7a8405f4a,34703,1733956857011, table=hbase:meta, region=1588230740 2024-12-11T22:41:04,138 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 56d7a8405f4a,34703,1733956857011, state=OPEN 2024-12-11T22:41:04,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,145 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,145 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,146 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,146 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN to state=OFFLINE, location=null, table=test, region=b30c1b01a8241f23347f03f93df7f090 to restore RIT 2024-12-11T22:41:04,146 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN to state=OFFLINE, location=null, table=test, region=f219e06826a4a3ede110adacdf28f6f4 to restore RIT 2024-12-11T22:41:04,146 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN to state=OFFLINE, location=null, table=test, region=5a93ec512c97cf46e8942b91f4c8f088 to restore RIT 2024-12-11T22:41:04,146 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=419f7368f61981f0a715c20304b74e0f, ASSIGN to state=OFFLINE, location=null, table=test, region=419f7368f61981f0a715c20304b74e0f to restore RIT 2024-12-11T22:41:04,147 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733956894147 2024-12-11T22:41:04,147 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-11T22:41:04,147 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:04,148 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-11T22:41:04,152 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a93ec512c97cf46e8942b91f4c8f088, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,152 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66aac9e74e8263b0b8914fdc86eec66c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,152 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-11T22:41:04,152 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6801ac4c1cb1416e045d462c592d2dc5, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,152 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN 2024-12-11T22:41:04,152 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:04,152 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f169feb3190772cec56c0ae51d57c26, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,152 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:04,154 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9e94fba4d03035b5feaab607eb13853, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,154 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419f7368f61981f0a715c20304b74e0f, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,154 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de3803c768bc063bdc71e495af58eb48, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,154 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b31ee44b6609c355adc37986b13d239c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,154 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00fd1046c94e7f507c0ee43e6555d034, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,154 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad4bb3db7f3f05f24b6115fc887d6ab5, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,154 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6561ead2eaa9f60da54d4ccff5474a28, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,154 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445df7fd625f94e0eb0712fcd777ebda, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,155 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f666d29f964372d8719d169fcecef641, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=5 2024-12-11T22:41:04,155 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f30d8a32c00877e2cf6f22e7f50620, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,155 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1a38f11b48b53983535b863fd4392ff, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,155 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01df3c1c862de5abd063d235e117a115, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,155 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f551d316d038ca1e1403bd05f890572, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,155 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f219e06826a4a3ede110adacdf28f6f4, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,155 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=daa100e1546a41747fac83159daee190, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,156 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da518288db554a985096494f24adc59f, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,156 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7dbe11f45ae33a883424947e613c7c6, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,156 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b30c1b01a8241f23347f03f93df7f090, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,156 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860d604009b38831efaa4f23b2d17596, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,156 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0e02a7a4f8d57169d1f435eabc00839, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,156 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f19f540286846a30911c41a83b97457e, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,156 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff9e1faa3192d62c84407efca19e0a89, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,156 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-11T22:41:04,156 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733956924156 2024-12-11T22:41:04,156 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733956984156 2024-12-11T22:41:04,156 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-12-11T22:41:04,156 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-11T22:41:04,156 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:04,156 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-11T22:41:04,157 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-11T22:41:04,157 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-11T22:41:04,157 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-11T22:41:04,157 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-11T22:41:04,157 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-11T22:41:04,158 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-11T22:41:04,160 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-11T22:41:04,160 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-11T22:41:04,161 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-11T22:41:04,161 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,161 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,161 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,161 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-11T22:41:04,161 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-11T22:41:04,161 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,161 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:04,161 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-11T22:41:04,162 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-11T22:41:04,162 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f666d29f964372d8719d169fcecef641, REOPEN/MOVE 2024-12-11T22:41:04,163 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true 2024-12-11T22:41:04,163 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,163 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,163 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-11T22:41:04,163 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:04,163 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,163 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,163 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN 2024-12-11T22:41:04,163 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN 2024-12-11T22:41:04,163 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN 2024-12-11T22:41:04,163 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,163 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,163 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,163 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,163 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,34703,1733956857011 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,163 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,163 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,163 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,163 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN 2024-12-11T22:41:04,164 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,164 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN 2024-12-11T22:41:04,164 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,164 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN 2024-12-11T22:41:04,164 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-11T22:41:04,165 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=65 2024-12-11T22:41:04,165 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=56d7a8405f4a,34703,1733956857011, table=hbase:meta, region=1588230740 2024-12-11T22:41:04,165 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 56d7a8405f4a,34703,1733956857011, state=OPEN 2024-12-11T22:41:04,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,170 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,170 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,170 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,171 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN to state=OFFLINE, location=null, table=test, region=b30c1b01a8241f23347f03f93df7f090 to restore RIT 2024-12-11T22:41:04,171 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN to state=OFFLINE, location=null, table=test, region=f219e06826a4a3ede110adacdf28f6f4 to restore RIT 2024-12-11T22:41:04,171 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5a93ec512c97cf46e8942b91f4c8f088, ASSIGN to state=OFFLINE, location=null, table=test, region=5a93ec512c97cf46e8942b91f4c8f088 to restore RIT 2024-12-11T22:41:04,171 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733956894171 2024-12-11T22:41:04,171 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-11T22:41:04,171 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:04,172 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-11T22:41:04,177 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-11T22:41:04,177 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN 2024-12-11T22:41:04,177 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:04,177 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:04,178 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a93ec512c97cf46e8942b91f4c8f088, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,178 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66aac9e74e8263b0b8914fdc86eec66c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,178 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6801ac4c1cb1416e045d462c592d2dc5, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,178 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f169feb3190772cec56c0ae51d57c26, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,178 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9e94fba4d03035b5feaab607eb13853, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,178 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419f7368f61981f0a715c20304b74e0f, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,178 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de3803c768bc063bdc71e495af58eb48, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,178 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b31ee44b6609c355adc37986b13d239c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,179 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00fd1046c94e7f507c0ee43e6555d034, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,179 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad4bb3db7f3f05f24b6115fc887d6ab5, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,179 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6561ead2eaa9f60da54d4ccff5474a28, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,179 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445df7fd625f94e0eb0712fcd777ebda, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,179 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f666d29f964372d8719d169fcecef641, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=5 2024-12-11T22:41:04,179 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f30d8a32c00877e2cf6f22e7f50620, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,179 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1a38f11b48b53983535b863fd4392ff, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,179 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01df3c1c862de5abd063d235e117a115, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,179 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f551d316d038ca1e1403bd05f890572, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,179 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f219e06826a4a3ede110adacdf28f6f4, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,179 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=daa100e1546a41747fac83159daee190, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,180 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da518288db554a985096494f24adc59f, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,180 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7dbe11f45ae33a883424947e613c7c6, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,180 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b30c1b01a8241f23347f03f93df7f090, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,180 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860d604009b38831efaa4f23b2d17596, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,180 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0e02a7a4f8d57169d1f435eabc00839, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,180 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f19f540286846a30911c41a83b97457e, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,180 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff9e1faa3192d62c84407efca19e0a89, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,180 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-11T22:41:04,180 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733956924180 2024-12-11T22:41:04,180 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733956984180 2024-12-11T22:41:04,180 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-12-11T22:41:04,180 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-11T22:41:04,180 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:04,180 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-11T22:41:04,180 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-11T22:41:04,181 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-11T22:41:04,181 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-11T22:41:04,181 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-11T22:41:04,181 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-11T22:41:04,182 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-11T22:41:04,183 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-11T22:41:04,183 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-11T22:41:04,183 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-11T22:41:04,183 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,183 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,183 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-11T22:41:04,183 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-11T22:41:04,183 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,183 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:04,183 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-11T22:41:04,184 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-11T22:41:04,184 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f666d29f964372d8719d169fcecef641, REOPEN/MOVE 2024-12-11T22:41:04,185 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true 2024-12-11T22:41:04,185 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,185 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,185 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-11T22:41:04,185 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:04,185 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,185 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,185 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN 2024-12-11T22:41:04,186 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN 2024-12-11T22:41:04,186 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,186 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,186 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,186 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,186 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,34703,1733956857011 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,186 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,186 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,186 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,186 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN 2024-12-11T22:41:04,186 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,186 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN 2024-12-11T22:41:04,186 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-11T22:41:04,187 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=65 2024-12-11T22:41:04,187 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=56d7a8405f4a,34703,1733956857011, table=hbase:meta, region=1588230740 2024-12-11T22:41:04,187 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 56d7a8405f4a,34703,1733956857011, state=OPEN 2024-12-11T22:41:04,197 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34843 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:04,197 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34843 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:04,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,214 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,214 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,214 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,215 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN to state=OFFLINE, location=null, table=test, region=b30c1b01a8241f23347f03f93df7f090 to restore RIT 2024-12-11T22:41:04,215 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f219e06826a4a3ede110adacdf28f6f4, ASSIGN to state=OFFLINE, location=null, table=test, region=f219e06826a4a3ede110adacdf28f6f4 to restore RIT 2024-12-11T22:41:04,215 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733956894215 2024-12-11T22:41:04,215 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-11T22:41:04,215 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:04,217 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-11T22:41:04,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a93ec512c97cf46e8942b91f4c8f088, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66aac9e74e8263b0b8914fdc86eec66c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,220 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-11T22:41:04,220 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN 2024-12-11T22:41:04,220 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:04,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6801ac4c1cb1416e045d462c592d2dc5, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f169feb3190772cec56c0ae51d57c26, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9e94fba4d03035b5feaab607eb13853, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,220 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:04,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419f7368f61981f0a715c20304b74e0f, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de3803c768bc063bdc71e495af58eb48, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b31ee44b6609c355adc37986b13d239c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00fd1046c94e7f507c0ee43e6555d034, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad4bb3db7f3f05f24b6115fc887d6ab5, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6561ead2eaa9f60da54d4ccff5474a28, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445df7fd625f94e0eb0712fcd777ebda, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f666d29f964372d8719d169fcecef641, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=5 2024-12-11T22:41:04,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f30d8a32c00877e2cf6f22e7f50620, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1a38f11b48b53983535b863fd4392ff, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01df3c1c862de5abd063d235e117a115, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f551d316d038ca1e1403bd05f890572, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f219e06826a4a3ede110adacdf28f6f4, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=daa100e1546a41747fac83159daee190, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da518288db554a985096494f24adc59f, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7dbe11f45ae33a883424947e613c7c6, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b30c1b01a8241f23347f03f93df7f090, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860d604009b38831efaa4f23b2d17596, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0e02a7a4f8d57169d1f435eabc00839, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f19f540286846a30911c41a83b97457e, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff9e1faa3192d62c84407efca19e0a89, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,222 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-11T22:41:04,222 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733956924222 2024-12-11T22:41:04,222 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733956984222 2024-12-11T22:41:04,222 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-11T22:41:04,222 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-11T22:41:04,222 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:04,223 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-11T22:41:04,223 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-11T22:41:04,223 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-11T22:41:04,223 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-11T22:41:04,223 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-11T22:41:04,223 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-11T22:41:04,224 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-11T22:41:04,225 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-11T22:41:04,226 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-11T22:41:04,226 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-11T22:41:04,226 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,226 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-11T22:41:04,226 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-11T22:41:04,226 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,226 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:04,226 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-11T22:41:04,227 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-11T22:41:04,227 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f666d29f964372d8719d169fcecef641, REOPEN/MOVE 2024-12-11T22:41:04,228 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true 2024-12-11T22:41:04,228 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,228 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,228 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-11T22:41:04,228 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:04,228 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,228 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,228 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN 2024-12-11T22:41:04,228 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,228 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,228 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,228 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,228 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,34703,1733956857011 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,228 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,228 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,228 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,228 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN 2024-12-11T22:41:04,228 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-11T22:41:04,229 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=65 2024-12-11T22:41:04,229 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=56d7a8405f4a,34703,1733956857011, table=hbase:meta, region=1588230740 2024-12-11T22:41:04,229 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 56d7a8405f4a,34703,1733956857011, state=OPEN 2024-12-11T22:41:04,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,245 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,245 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,245 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,246 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b30c1b01a8241f23347f03f93df7f090, ASSIGN to state=OFFLINE, location=null, table=test, region=b30c1b01a8241f23347f03f93df7f090 to restore RIT 2024-12-11T22:41:04,246 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733956894246 2024-12-11T22:41:04,246 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-11T22:41:04,246 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:04,248 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-11T22:41:04,252 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-11T22:41:04,252 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,252 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:04,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a93ec512c97cf46e8942b91f4c8f088, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,252 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:04,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66aac9e74e8263b0b8914fdc86eec66c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6801ac4c1cb1416e045d462c592d2dc5, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f169feb3190772cec56c0ae51d57c26, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9e94fba4d03035b5feaab607eb13853, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419f7368f61981f0a715c20304b74e0f, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de3803c768bc063bdc71e495af58eb48, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b31ee44b6609c355adc37986b13d239c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00fd1046c94e7f507c0ee43e6555d034, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad4bb3db7f3f05f24b6115fc887d6ab5, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6561ead2eaa9f60da54d4ccff5474a28, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445df7fd625f94e0eb0712fcd777ebda, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f666d29f964372d8719d169fcecef641, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=5 2024-12-11T22:41:04,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f30d8a32c00877e2cf6f22e7f50620, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1a38f11b48b53983535b863fd4392ff, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01df3c1c862de5abd063d235e117a115, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f551d316d038ca1e1403bd05f890572, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f219e06826a4a3ede110adacdf28f6f4, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=daa100e1546a41747fac83159daee190, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da518288db554a985096494f24adc59f, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7dbe11f45ae33a883424947e613c7c6, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b30c1b01a8241f23347f03f93df7f090, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860d604009b38831efaa4f23b2d17596, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0e02a7a4f8d57169d1f435eabc00839, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f19f540286846a30911c41a83b97457e, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff9e1faa3192d62c84407efca19e0a89, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,254 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-11T22:41:04,254 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733956924254 2024-12-11T22:41:04,254 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733956984254 2024-12-11T22:41:04,254 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-11T22:41:04,254 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-11T22:41:04,254 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:04,254 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-11T22:41:04,254 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-11T22:41:04,255 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-11T22:41:04,255 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-11T22:41:04,255 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-11T22:41:04,255 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-11T22:41:04,256 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-11T22:41:04,257 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-11T22:41:04,257 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-11T22:41:04,257 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-11T22:41:04,257 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-11T22:41:04,257 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-11T22:41:04,257 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,257 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:04,257 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-11T22:41:04,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-11T22:41:04,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f666d29f964372d8719d169fcecef641, REOPEN/MOVE 2024-12-11T22:41:04,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true 2024-12-11T22:41:04,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-11T22:41:04,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:04,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,258 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,258 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,258 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,258 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,258 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,34703,1733956857011 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,259 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,259 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,259 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-12-11T22:41:04,259 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=65 2024-12-11T22:41:04,259 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=56d7a8405f4a,34703,1733956857011, table=hbase:meta, region=1588230740 2024-12-11T22:41:04,260 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 56d7a8405f4a,34703,1733956857011, state=OPEN 2024-12-11T22:41:04,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,270 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,270 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,270 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,271 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733956894271 2024-12-11T22:41:04,272 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-11T22:41:04,272 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:04,272 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-11T22:41:04,274 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-11T22:41:04,274 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:04,274 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:04,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a93ec512c97cf46e8942b91f4c8f088, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66aac9e74e8263b0b8914fdc86eec66c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6801ac4c1cb1416e045d462c592d2dc5, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f169feb3190772cec56c0ae51d57c26, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9e94fba4d03035b5feaab607eb13853, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419f7368f61981f0a715c20304b74e0f, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de3803c768bc063bdc71e495af58eb48, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b31ee44b6609c355adc37986b13d239c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00fd1046c94e7f507c0ee43e6555d034, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad4bb3db7f3f05f24b6115fc887d6ab5, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6561ead2eaa9f60da54d4ccff5474a28, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445df7fd625f94e0eb0712fcd777ebda, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f666d29f964372d8719d169fcecef641, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=5 2024-12-11T22:41:04,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f30d8a32c00877e2cf6f22e7f50620, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1a38f11b48b53983535b863fd4392ff, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01df3c1c862de5abd063d235e117a115, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f551d316d038ca1e1403bd05f890572, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f219e06826a4a3ede110adacdf28f6f4, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=daa100e1546a41747fac83159daee190, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da518288db554a985096494f24adc59f, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7dbe11f45ae33a883424947e613c7c6, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b30c1b01a8241f23347f03f93df7f090, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860d604009b38831efaa4f23b2d17596, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0e02a7a4f8d57169d1f435eabc00839, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f19f540286846a30911c41a83b97457e, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff9e1faa3192d62c84407efca19e0a89, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,278 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-11T22:41:04,278 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733956924278 2024-12-11T22:41:04,278 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733956984278 2024-12-11T22:41:04,278 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-12-11T22:41:04,278 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-11T22:41:04,278 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:04,278 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-11T22:41:04,278 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-11T22:41:04,278 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-11T22:41:04,278 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-11T22:41:04,278 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-11T22:41:04,278 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-11T22:41:04,279 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-11T22:41:04,280 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-11T22:41:04,280 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-11T22:41:04,280 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-11T22:41:04,280 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-11T22:41:04,280 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,280 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:04,280 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-11T22:41:04,281 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-11T22:41:04,281 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f666d29f964372d8719d169fcecef641, REOPEN/MOVE 2024-12-11T22:41:04,281 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true 2024-12-11T22:41:04,281 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,282 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,282 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-11T22:41:04,282 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:04,282 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,282 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,282 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,282 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,282 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,282 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,34703,1733956857011 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,282 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.1733956858632, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,282 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-12-11T22:41:04,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=65 2024-12-11T22:41:04,283 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=56d7a8405f4a,34703,1733956857011, table=hbase:meta, region=1588230740 2024-12-11T22:41:04,283 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 56d7a8405f4a,34703,1733956857011, state=OPEN 2024-12-11T22:41:04,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,287 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,287 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,287 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,288 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733956894288 2024-12-11T22:41:04,288 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-11T22:41:04,288 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:04,289 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-11T22:41:04,289 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-11T22:41:04,290 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:04,290 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:04,290 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,34703,1733956857011 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:04,292 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a93ec512c97cf46e8942b91f4c8f088, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,292 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66aac9e74e8263b0b8914fdc86eec66c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,293 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6801ac4c1cb1416e045d462c592d2dc5, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,293 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f169feb3190772cec56c0ae51d57c26, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,293 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9e94fba4d03035b5feaab607eb13853, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,293 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419f7368f61981f0a715c20304b74e0f, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,293 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de3803c768bc063bdc71e495af58eb48, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,293 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b31ee44b6609c355adc37986b13d239c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,293 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00fd1046c94e7f507c0ee43e6555d034, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,293 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad4bb3db7f3f05f24b6115fc887d6ab5, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,293 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6561ead2eaa9f60da54d4ccff5474a28, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,293 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445df7fd625f94e0eb0712fcd777ebda, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,293 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f666d29f964372d8719d169fcecef641, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=5 2024-12-11T22:41:04,293 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f30d8a32c00877e2cf6f22e7f50620, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,293 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1a38f11b48b53983535b863fd4392ff, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,293 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01df3c1c862de5abd063d235e117a115, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,294 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f551d316d038ca1e1403bd05f890572, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,294 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f219e06826a4a3ede110adacdf28f6f4, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,294 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=daa100e1546a41747fac83159daee190, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,294 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da518288db554a985096494f24adc59f, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,294 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7dbe11f45ae33a883424947e613c7c6, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,294 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b30c1b01a8241f23347f03f93df7f090, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,294 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860d604009b38831efaa4f23b2d17596, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,294 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0e02a7a4f8d57169d1f435eabc00839, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,294 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f19f540286846a30911c41a83b97457e, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,294 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff9e1faa3192d62c84407efca19e0a89, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,294 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-11T22:41:04,294 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733956924294 2024-12-11T22:41:04,294 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733956984294 2024-12-11T22:41:04,294 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-12-11T22:41:04,294 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-11T22:41:04,294 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:04,294 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-11T22:41:04,294 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-11T22:41:04,295 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-11T22:41:04,295 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-11T22:41:04,295 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-11T22:41:04,295 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-11T22:41:04,296 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-11T22:41:04,298 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-11T22:41:04,298 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-11T22:41:04,298 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-11T22:41:04,298 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,298 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-11T22:41:04,298 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-11T22:41:04,299 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-11T22:41:04,299 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f666d29f964372d8719d169fcecef641, REOPEN/MOVE 2024-12-11T22:41:04,299 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true 2024-12-11T22:41:04,300 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,300 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,300 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-11T22:41:04,300 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:04,300 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,300 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,300 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,300 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,300 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=56d7a8405f4a,34703,1733956857011 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,300 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-11T22:41:04,301 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=65 2024-12-11T22:41:04,301 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=56d7a8405f4a,34703,1733956857011, table=hbase:meta, region=1588230740 2024-12-11T22:41:04,301 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 56d7a8405f4a,34703,1733956857011, state=OPEN 2024-12-11T22:41:04,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,312 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,312 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,312 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,313 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733956894313 2024-12-11T22:41:04,313 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-11T22:41:04,313 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:04,314 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-11T22:41:04,315 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-11T22:41:04,315 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-11T22:41:04,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:04,316 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:04,318 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a93ec512c97cf46e8942b91f4c8f088, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,318 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66aac9e74e8263b0b8914fdc86eec66c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,318 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6801ac4c1cb1416e045d462c592d2dc5, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,318 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f169feb3190772cec56c0ae51d57c26, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,318 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9e94fba4d03035b5feaab607eb13853, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,318 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419f7368f61981f0a715c20304b74e0f, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,318 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de3803c768bc063bdc71e495af58eb48, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,318 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b31ee44b6609c355adc37986b13d239c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,318 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00fd1046c94e7f507c0ee43e6555d034, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,318 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad4bb3db7f3f05f24b6115fc887d6ab5, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,319 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6561ead2eaa9f60da54d4ccff5474a28, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,319 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445df7fd625f94e0eb0712fcd777ebda, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,319 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f666d29f964372d8719d169fcecef641, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=5 2024-12-11T22:41:04,319 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f30d8a32c00877e2cf6f22e7f50620, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,319 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1a38f11b48b53983535b863fd4392ff, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,319 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01df3c1c862de5abd063d235e117a115, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,319 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f551d316d038ca1e1403bd05f890572, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,319 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f219e06826a4a3ede110adacdf28f6f4, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,319 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=daa100e1546a41747fac83159daee190, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,319 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da518288db554a985096494f24adc59f, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,319 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7dbe11f45ae33a883424947e613c7c6, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,319 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b30c1b01a8241f23347f03f93df7f090, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,319 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860d604009b38831efaa4f23b2d17596, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,319 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0e02a7a4f8d57169d1f435eabc00839, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f19f540286846a30911c41a83b97457e, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff9e1faa3192d62c84407efca19e0a89, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,320 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-11T22:41:04,320 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733956924320 2024-12-11T22:41:04,320 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733956984320 2024-12-11T22:41:04,320 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-12-11T22:41:04,320 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-11T22:41:04,320 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:04,320 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-11T22:41:04,320 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-11T22:41:04,320 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-11T22:41:04,320 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-11T22:41:04,320 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-11T22:41:04,321 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-11T22:41:04,321 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-11T22:41:04,322 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-11T22:41:04,322 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-11T22:41:04,322 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-11T22:41:04,322 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-11T22:41:04,322 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-11T22:41:04,323 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-11T22:41:04,323 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f666d29f964372d8719d169fcecef641, REOPEN/MOVE 2024-12-11T22:41:04,323 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true 2024-12-11T22:41:04,323 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,324 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,324 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-11T22:41:04,324 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,324 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,324 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,324 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,324 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-12-11T22:41:04,324 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=65 2024-12-11T22:41:04,324 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=56d7a8405f4a,34703,1733956857011, table=hbase:meta, region=1588230740 2024-12-11T22:41:04,325 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 56d7a8405f4a,34703,1733956857011, state=OPEN 2024-12-11T22:41:04,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,329 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,329 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,329 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,330 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733956894330 2024-12-11T22:41:04,330 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-11T22:41:04,330 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:04,331 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-11T22:41:04,333 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-11T22:41:04,333 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,333 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:04,334 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a93ec512c97cf46e8942b91f4c8f088, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,333 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:04,334 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66aac9e74e8263b0b8914fdc86eec66c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,334 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6801ac4c1cb1416e045d462c592d2dc5, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,334 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f169feb3190772cec56c0ae51d57c26, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,334 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9e94fba4d03035b5feaab607eb13853, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,334 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419f7368f61981f0a715c20304b74e0f, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,334 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de3803c768bc063bdc71e495af58eb48, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,334 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b31ee44b6609c355adc37986b13d239c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,334 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00fd1046c94e7f507c0ee43e6555d034, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,334 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad4bb3db7f3f05f24b6115fc887d6ab5, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,334 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6561ead2eaa9f60da54d4ccff5474a28, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,334 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445df7fd625f94e0eb0712fcd777ebda, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,335 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f666d29f964372d8719d169fcecef641, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=5 2024-12-11T22:41:04,335 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f30d8a32c00877e2cf6f22e7f50620, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,335 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1a38f11b48b53983535b863fd4392ff, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,335 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01df3c1c862de5abd063d235e117a115, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,335 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f551d316d038ca1e1403bd05f890572, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,335 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f219e06826a4a3ede110adacdf28f6f4, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,335 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=daa100e1546a41747fac83159daee190, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,335 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da518288db554a985096494f24adc59f, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,335 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7dbe11f45ae33a883424947e613c7c6, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,335 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b30c1b01a8241f23347f03f93df7f090, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,335 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860d604009b38831efaa4f23b2d17596, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,335 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0e02a7a4f8d57169d1f435eabc00839, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,335 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f19f540286846a30911c41a83b97457e, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,335 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff9e1faa3192d62c84407efca19e0a89, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,336 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-11T22:41:04,336 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733956924336 2024-12-11T22:41:04,336 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733956984336 2024-12-11T22:41:04,336 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec 2024-12-11T22:41:04,336 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-11T22:41:04,336 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:04,336 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-11T22:41:04,336 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-11T22:41:04,336 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-11T22:41:04,336 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-11T22:41:04,336 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-11T22:41:04,336 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-11T22:41:04,337 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-11T22:41:04,338 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-11T22:41:04,338 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-11T22:41:04,338 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-11T22:41:04,338 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-11T22:41:04,339 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-11T22:41:04,339 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f666d29f964372d8719d169fcecef641, REOPEN/MOVE 2024-12-11T22:41:04,339 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true 2024-12-11T22:41:04,339 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,339 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,340 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,340 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,340 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,340 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-12-11T22:41:04,340 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=65 2024-12-11T22:41:04,340 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=56d7a8405f4a,34703,1733956857011, table=hbase:meta, region=1588230740 2024-12-11T22:41:04,341 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 56d7a8405f4a,34703,1733956857011, state=OPEN 2024-12-11T22:41:04,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,345 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,345 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,345 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,347 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733956894347 2024-12-11T22:41:04,347 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-11T22:41:04,347 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:04,348 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-11T22:41:04,349 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-11T22:41:04,349 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,349 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:04,349 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:04,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a93ec512c97cf46e8942b91f4c8f088, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66aac9e74e8263b0b8914fdc86eec66c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6801ac4c1cb1416e045d462c592d2dc5, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f169feb3190772cec56c0ae51d57c26, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9e94fba4d03035b5feaab607eb13853, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419f7368f61981f0a715c20304b74e0f, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de3803c768bc063bdc71e495af58eb48, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b31ee44b6609c355adc37986b13d239c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00fd1046c94e7f507c0ee43e6555d034, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad4bb3db7f3f05f24b6115fc887d6ab5, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6561ead2eaa9f60da54d4ccff5474a28, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445df7fd625f94e0eb0712fcd777ebda, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f666d29f964372d8719d169fcecef641, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=5 2024-12-11T22:41:04,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f30d8a32c00877e2cf6f22e7f50620, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1a38f11b48b53983535b863fd4392ff, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01df3c1c862de5abd063d235e117a115, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f551d316d038ca1e1403bd05f890572, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f219e06826a4a3ede110adacdf28f6f4, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=daa100e1546a41747fac83159daee190, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da518288db554a985096494f24adc59f, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7dbe11f45ae33a883424947e613c7c6, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b30c1b01a8241f23347f03f93df7f090, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860d604009b38831efaa4f23b2d17596, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0e02a7a4f8d57169d1f435eabc00839, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,354 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f19f540286846a30911c41a83b97457e, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,354 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff9e1faa3192d62c84407efca19e0a89, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,354 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-11T22:41:04,354 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733956924354 2024-12-11T22:41:04,354 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733956984354 2024-12-11T22:41:04,354 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-11T22:41:04,354 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-11T22:41:04,354 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:04,354 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-11T22:41:04,354 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-11T22:41:04,354 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-11T22:41:04,354 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-11T22:41:04,354 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-11T22:41:04,355 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-11T22:41:04,355 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-11T22:41:04,356 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-11T22:41:04,356 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-11T22:41:04,356 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-11T22:41:04,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-11T22:41:04,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f666d29f964372d8719d169fcecef641, REOPEN/MOVE 2024-12-11T22:41:04,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true 2024-12-11T22:41:04,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,357 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,357 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 56d7a8405f4a%2C36971%2C1733956857186.meta.1733956859111.meta, worker=56d7a8405f4a,40421,1733956857131 didn't hold the lock before restarting, skip acquiring lock. 2024-12-11T22:41:04,357 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-12-11T22:41:04,358 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=65 2024-12-11T22:41:04,358 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=56d7a8405f4a,34703,1733956857011, table=hbase:meta, region=1588230740 2024-12-11T22:41:04,358 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 56d7a8405f4a,34703,1733956857011, state=OPEN 2024-12-11T22:41:04,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,370 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,370 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,370 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,371 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733956894370 2024-12-11T22:41:04,371 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-11T22:41:04,371 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:04,372 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-11T22:41:04,373 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-11T22:41:04,373 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true 2024-12-11T22:41:04,373 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:04,373 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:04,375 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a93ec512c97cf46e8942b91f4c8f088, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,375 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66aac9e74e8263b0b8914fdc86eec66c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,375 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6801ac4c1cb1416e045d462c592d2dc5, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,375 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f169feb3190772cec56c0ae51d57c26, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,375 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9e94fba4d03035b5feaab607eb13853, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,375 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419f7368f61981f0a715c20304b74e0f, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,375 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de3803c768bc063bdc71e495af58eb48, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,375 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b31ee44b6609c355adc37986b13d239c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,375 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00fd1046c94e7f507c0ee43e6555d034, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,375 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad4bb3db7f3f05f24b6115fc887d6ab5, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,376 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6561ead2eaa9f60da54d4ccff5474a28, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,376 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445df7fd625f94e0eb0712fcd777ebda, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,376 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f666d29f964372d8719d169fcecef641, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=5 2024-12-11T22:41:04,376 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f30d8a32c00877e2cf6f22e7f50620, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,376 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1a38f11b48b53983535b863fd4392ff, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,376 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01df3c1c862de5abd063d235e117a115, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,376 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f551d316d038ca1e1403bd05f890572, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,376 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f219e06826a4a3ede110adacdf28f6f4, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,376 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=daa100e1546a41747fac83159daee190, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,376 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da518288db554a985096494f24adc59f, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,376 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7dbe11f45ae33a883424947e613c7c6, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,376 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b30c1b01a8241f23347f03f93df7f090, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,376 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860d604009b38831efaa4f23b2d17596, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,376 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0e02a7a4f8d57169d1f435eabc00839, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,376 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f19f540286846a30911c41a83b97457e, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,376 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff9e1faa3192d62c84407efca19e0a89, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,377 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-11T22:41:04,377 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733956924377 2024-12-11T22:41:04,377 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733956984377 2024-12-11T22:41:04,377 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-12-11T22:41:04,377 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-11T22:41:04,377 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:04,377 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-11T22:41:04,377 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-11T22:41:04,377 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-11T22:41:04,377 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-11T22:41:04,377 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-11T22:41:04,377 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-11T22:41:04,378 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-11T22:41:04,379 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-11T22:41:04,379 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-11T22:41:04,380 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-11T22:41:04,380 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f666d29f964372d8719d169fcecef641, REOPEN/MOVE 2024-12-11T22:41:04,380 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true 2024-12-11T22:41:04,380 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-11T22:41:04,380 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-12-11T22:41:04,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=65 2024-12-11T22:41:04,381 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=56d7a8405f4a,34703,1733956857011, table=hbase:meta, region=1588230740 2024-12-11T22:41:04,381 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 56d7a8405f4a,34703,1733956857011, state=OPEN 2024-12-11T22:41:04,387 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,387 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,387 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:41:04,387 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,387 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,387 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:41:04,388 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733956894388 2024-12-11T22:41:04,388 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-11T22:41:04,388 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:41:04,389 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-11T22:41:04,390 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=60, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 5a93ec512c97cf46e8942b91f4c8f088, server=56d7a8405f4a,34703,1733956857011:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 56d7a8405f4a,36971,1733956857186, splitWal=true, meta=true exec-time=2.4420 sec 2024-12-11T22:41:04,392 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a93ec512c97cf46e8942b91f4c8f088, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,393 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66aac9e74e8263b0b8914fdc86eec66c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,393 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6801ac4c1cb1416e045d462c592d2dc5, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,393 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9f169feb3190772cec56c0ae51d57c26, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,393 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e9e94fba4d03035b5feaab607eb13853, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,393 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=419f7368f61981f0a715c20304b74e0f, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,393 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de3803c768bc063bdc71e495af58eb48, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,393 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b31ee44b6609c355adc37986b13d239c, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,393 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00fd1046c94e7f507c0ee43e6555d034, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,393 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad4bb3db7f3f05f24b6115fc887d6ab5, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,394 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6561ead2eaa9f60da54d4ccff5474a28, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,394 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=445df7fd625f94e0eb0712fcd777ebda, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,394 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f666d29f964372d8719d169fcecef641, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=5 2024-12-11T22:41:04,394 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=65f30d8a32c00877e2cf6f22e7f50620, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,394 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1a38f11b48b53983535b863fd4392ff, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,394 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01df3c1c862de5abd063d235e117a115, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,394 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f551d316d038ca1e1403bd05f890572, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,394 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f219e06826a4a3ede110adacdf28f6f4, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,394 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=daa100e1546a41747fac83159daee190, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,394 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da518288db554a985096494f24adc59f, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,395 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7dbe11f45ae33a883424947e613c7c6, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,395 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b30c1b01a8241f23347f03f93df7f090, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,395 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=860d604009b38831efaa4f23b2d17596, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,395 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0e02a7a4f8d57169d1f435eabc00839, regionState=OPENING, lastHost=56d7a8405f4a,36971,1733956857186, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,395 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f19f540286846a30911c41a83b97457e, regionState=OPEN, lastHost=56d7a8405f4a,34703,1733956857011, regionLocation=56d7a8405f4a,34703,1733956857011, openSeqNum=2 2024-12-11T22:41:04,395 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff9e1faa3192d62c84407efca19e0a89, regionState=OPEN, lastHost=56d7a8405f4a,40421,1733956857131, regionLocation=56d7a8405f4a,40421,1733956857131, openSeqNum=2 2024-12-11T22:41:04,395 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-11T22:41:04,395 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733956924395 2024-12-11T22:41:04,395 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733956984395 2024-12-11T22:41:04,395 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-11T22:41:04,410 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=350 (was 384), OpenFileDescriptor=622 (was 643), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=172 (was 172), ProcessCount=11 (was 11), AvailableMemoryMB=6024 (was 6075) 2024-12-11T22:41:04,410 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-11T22:41:04,411 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-11T22:41:04,411 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T22:41:04,411 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T22:41:04,411 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T22:41:04,411 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T22:41:04,411 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-11T22:41:04,412 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=889085506, stopped=false 2024-12-11T22:41:04,412 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=56d7a8405f4a,34843,1733956856287 2024-12-11T22:41:04,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-11T22:41:04,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-11T22:41:04,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-11T22:41:04,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:41:04,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:41:04,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:41:04,420 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:04,420 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-11T22:41:04,421 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T22:41:04,421 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T22:41:04,421 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T22:41:04,421 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T22:41:04,421 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T22:41:04,421 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '56d7a8405f4a,34703,1733956857011' ***** 2024-12-11T22:41:04,421 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-11T22:41:04,421 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '56d7a8405f4a,40421,1733956857131' ***** 2024-12-11T22:41:04,421 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-11T22:41:04,422 INFO [RS:1;56d7a8405f4a:40421 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-11T22:41:04,422 INFO [RS:0;56d7a8405f4a:34703 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-11T22:41:04,422 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-11T22:41:04,422 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-11T22:41:04,422 INFO [RS:0;56d7a8405f4a:34703 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-11T22:41:04,422 INFO [RS:0;56d7a8405f4a:34703 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-11T22:41:04,422 INFO [RS:0;56d7a8405f4a:34703 {}] regionserver.HRegionServer(3091): Received CLOSE for daa100e1546a41747fac83159daee190 2024-12-11T22:41:04,422 INFO [RS:1;56d7a8405f4a:40421 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-11T22:41:04,422 INFO [RS:1;56d7a8405f4a:40421 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-11T22:41:04,422 INFO [RS:1;56d7a8405f4a:40421 {}] regionserver.HRegionServer(3091): Received CLOSE for 2f551d316d038ca1e1403bd05f890572 2024-12-11T22:41:04,423 INFO [RS:0;56d7a8405f4a:34703 {}] regionserver.HRegionServer(3091): Received CLOSE for f666d29f964372d8719d169fcecef641 2024-12-11T22:41:04,423 INFO [RS:0;56d7a8405f4a:34703 {}] regionserver.HRegionServer(3091): Received CLOSE for b30c1b01a8241f23347f03f93df7f090 2024-12-11T22:41:04,423 INFO [RS:1;56d7a8405f4a:40421 {}] regionserver.HRegionServer(3091): Received CLOSE for 860d604009b38831efaa4f23b2d17596 2024-12-11T22:41:04,423 INFO [RS:0;56d7a8405f4a:34703 {}] regionserver.HRegionServer(3091): Received CLOSE for 6561ead2eaa9f60da54d4ccff5474a28 2024-12-11T22:41:04,423 INFO [RS:1;56d7a8405f4a:40421 {}] regionserver.HRegionServer(3091): Received CLOSE for 419f7368f61981f0a715c20304b74e0f 2024-12-11T22:41:04,423 INFO [RS:0;56d7a8405f4a:34703 {}] regionserver.HRegionServer(3091): Received CLOSE for 00fd1046c94e7f507c0ee43e6555d034 2024-12-11T22:41:04,423 INFO [RS:0;56d7a8405f4a:34703 {}] regionserver.HRegionServer(3091): Received CLOSE for f19f540286846a30911c41a83b97457e 2024-12-11T22:41:04,423 INFO [RS:1;56d7a8405f4a:40421 {}] regionserver.HRegionServer(3091): Received CLOSE for ad4bb3db7f3f05f24b6115fc887d6ab5 2024-12-11T22:41:04,423 INFO [RS:0;56d7a8405f4a:34703 {}] regionserver.HRegionServer(3091): Received CLOSE for 5a93ec512c97cf46e8942b91f4c8f088 2024-12-11T22:41:04,423 INFO [RS:1;56d7a8405f4a:40421 {}] regionserver.HRegionServer(3091): Received CLOSE for 66aac9e74e8263b0b8914fdc86eec66c 2024-12-11T22:41:04,423 INFO [RS:0;56d7a8405f4a:34703 {}] regionserver.HRegionServer(3091): Received CLOSE for 6801ac4c1cb1416e045d462c592d2dc5 2024-12-11T22:41:04,423 INFO [RS:0;56d7a8405f4a:34703 {}] regionserver.HRegionServer(3091): Received CLOSE for f7dbe11f45ae33a883424947e613c7c6 2024-12-11T22:41:04,423 INFO [RS:1;56d7a8405f4a:40421 {}] regionserver.HRegionServer(3091): Received CLOSE for f219e06826a4a3ede110adacdf28f6f4 2024-12-11T22:41:04,423 INFO [RS:0;56d7a8405f4a:34703 {}] regionserver.HRegionServer(3091): Received CLOSE for 01df3c1c862de5abd063d235e117a115 2024-12-11T22:41:04,423 INFO [RS:1;56d7a8405f4a:40421 {}] regionserver.HRegionServer(3091): Received CLOSE for 445df7fd625f94e0eb0712fcd777ebda 2024-12-11T22:41:04,423 INFO [RS:0;56d7a8405f4a:34703 {}] regionserver.HRegionServer(3091): Received CLOSE for de3803c768bc063bdc71e495af58eb48 2024-12-11T22:41:04,423 INFO [RS:0;56d7a8405f4a:34703 {}] regionserver.HRegionServer(3091): Received CLOSE for b1a38f11b48b53983535b863fd4392ff 2024-12-11T22:41:04,423 INFO [RS:1;56d7a8405f4a:40421 {}] regionserver.HRegionServer(3091): Received CLOSE for 65f30d8a32c00877e2cf6f22e7f50620 2024-12-11T22:41:04,423 INFO [RS:0;56d7a8405f4a:34703 {}] regionserver.HRegionServer(3091): Received CLOSE for e9e94fba4d03035b5feaab607eb13853 2024-12-11T22:41:04,423 INFO [RS:1;56d7a8405f4a:40421 {}] regionserver.HRegionServer(3091): Received CLOSE for 9f169feb3190772cec56c0ae51d57c26 2024-12-11T22:41:04,423 INFO [RS:1;56d7a8405f4a:40421 {}] regionserver.HRegionServer(3091): Received CLOSE for da518288db554a985096494f24adc59f 2024-12-11T22:41:04,423 INFO [RS:1;56d7a8405f4a:40421 {}] regionserver.HRegionServer(3091): Received CLOSE for a0e02a7a4f8d57169d1f435eabc00839 2024-12-11T22:41:04,423 INFO [RS:0;56d7a8405f4a:34703 {}] regionserver.HRegionServer(959): stopping server 56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:04,423 INFO [RS:1;56d7a8405f4a:40421 {}] regionserver.HRegionServer(3091): Received CLOSE for ff9e1faa3192d62c84407efca19e0a89 2024-12-11T22:41:04,423 INFO [RS:0;56d7a8405f4a:34703 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-11T22:41:04,423 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2f551d316d038ca1e1403bd05f890572, disabling compactions & flushes 2024-12-11T22:41:04,423 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing daa100e1546a41747fac83159daee190, disabling compactions & flushes 2024-12-11T22:41:04,423 INFO [RS:1;56d7a8405f4a:40421 {}] regionserver.HRegionServer(3091): Received CLOSE for b31ee44b6609c355adc37986b13d239c 2024-12-11T22:41:04,424 INFO [RS:0;56d7a8405f4a:34703 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;56d7a8405f4a:34703. 2024-12-11T22:41:04,424 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1733956859502.2f551d316d038ca1e1403bd05f890572. 2024-12-11T22:41:04,424 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1733956859502.daa100e1546a41747fac83159daee190. 2024-12-11T22:41:04,424 INFO [RS:1;56d7a8405f4a:40421 {}] regionserver.HRegionServer(959): stopping server 56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,424 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1733956859502.2f551d316d038ca1e1403bd05f890572. 2024-12-11T22:41:04,424 INFO [RS:1;56d7a8405f4a:40421 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-11T22:41:04,424 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1733956859502.daa100e1546a41747fac83159daee190. 2024-12-11T22:41:04,424 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1733956859502.2f551d316d038ca1e1403bd05f890572. after waiting 0 ms 2024-12-11T22:41:04,424 DEBUG [RS:0;56d7a8405f4a:34703 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T22:41:04,424 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1733956859502.2f551d316d038ca1e1403bd05f890572. 2024-12-11T22:41:04,424 DEBUG [RS:0;56d7a8405f4a:34703 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T22:41:04,424 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1733956859502.daa100e1546a41747fac83159daee190. after waiting 0 ms 2024-12-11T22:41:04,424 INFO [RS:1;56d7a8405f4a:40421 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;56d7a8405f4a:40421. 2024-12-11T22:41:04,424 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1733956859502.daa100e1546a41747fac83159daee190. 2024-12-11T22:41:04,424 DEBUG [RS:1;56d7a8405f4a:40421 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T22:41:04,424 DEBUG [RS:1;56d7a8405f4a:40421 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T22:41:04,424 INFO [RS:0;56d7a8405f4a:34703 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-11T22:41:04,424 INFO [RS:0;56d7a8405f4a:34703 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-11T22:41:04,424 INFO [RS:0;56d7a8405f4a:34703 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-11T22:41:04,424 INFO [RS:1;56d7a8405f4a:40421 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-12-11T22:41:04,424 INFO [RS:0;56d7a8405f4a:34703 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-11T22:41:04,424 DEBUG [RS:1;56d7a8405f4a:40421 {}] regionserver.HRegionServer(1351): Waiting on 2f551d316d038ca1e1403bd05f890572, 419f7368f61981f0a715c20304b74e0f, 445df7fd625f94e0eb0712fcd777ebda, 65f30d8a32c00877e2cf6f22e7f50620, 66aac9e74e8263b0b8914fdc86eec66c, 860d604009b38831efaa4f23b2d17596, 9f169feb3190772cec56c0ae51d57c26, a0e02a7a4f8d57169d1f435eabc00839, ad4bb3db7f3f05f24b6115fc887d6ab5, b31ee44b6609c355adc37986b13d239c, da518288db554a985096494f24adc59f, f219e06826a4a3ede110adacdf28f6f4, ff9e1faa3192d62c84407efca19e0a89 2024-12-11T22:41:04,425 INFO [RS:0;56d7a8405f4a:34703 {}] regionserver.HRegionServer(1321): Waiting on 14 regions to close 2024-12-11T22:41:04,425 DEBUG [RS:0;56d7a8405f4a:34703 {}] regionserver.HRegionServer(1351): Waiting on 00fd1046c94e7f507c0ee43e6555d034, 01df3c1c862de5abd063d235e117a115, 1588230740, 5a93ec512c97cf46e8942b91f4c8f088, 6561ead2eaa9f60da54d4ccff5474a28, 6801ac4c1cb1416e045d462c592d2dc5, b1a38f11b48b53983535b863fd4392ff, b30c1b01a8241f23347f03f93df7f090, daa100e1546a41747fac83159daee190, de3803c768bc063bdc71e495af58eb48, e9e94fba4d03035b5feaab607eb13853, f19f540286846a30911c41a83b97457e, f666d29f964372d8719d169fcecef641, f7dbe11f45ae33a883424947e613c7c6 2024-12-11T22:41:04,425 DEBUG [RS_CLOSE_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-11T22:41:04,425 INFO [RS_CLOSE_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-11T22:41:04,425 DEBUG [RS_CLOSE_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-11T22:41:04,425 DEBUG [RS_CLOSE_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-11T22:41:04,425 DEBUG [RS_CLOSE_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-11T22:41:04,425 INFO [RS_CLOSE_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.64 KB heapSize=5.42 KB 2024-12-11T22:41:04,429 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/daa100e1546a41747fac83159daee190/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-11T22:41:04,430 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1733956859502.daa100e1546a41747fac83159daee190. 2024-12-11T22:41:04,430 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for daa100e1546a41747fac83159daee190: Waiting for close lock at 1733956864423Running coprocessor pre-close hooks at 1733956864423Disabling compacts and flushes for region at 1733956864423Disabling writes for close at 1733956864424 (+1 ms)Writing region close event to WAL at 1733956864424Running coprocessor post-close hooks at 1733956864430 (+6 ms)Closed at 1733956864430 2024-12-11T22:41:04,430 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/2f551d316d038ca1e1403bd05f890572/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-11T22:41:04,430 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1733956859502.daa100e1546a41747fac83159daee190. 2024-12-11T22:41:04,431 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f666d29f964372d8719d169fcecef641, disabling compactions & flushes 2024-12-11T22:41:04,431 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1733956859502.f666d29f964372d8719d169fcecef641. 2024-12-11T22:41:04,431 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1733956859502.f666d29f964372d8719d169fcecef641. 2024-12-11T22:41:04,431 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1733956859502.f666d29f964372d8719d169fcecef641. after waiting 0 ms 2024-12-11T22:41:04,431 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1733956859502.f666d29f964372d8719d169fcecef641. 2024-12-11T22:41:04,432 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1733956859502.2f551d316d038ca1e1403bd05f890572. 2024-12-11T22:41:04,432 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2f551d316d038ca1e1403bd05f890572: Waiting for close lock at 1733956864423Running coprocessor pre-close hooks at 1733956864423Disabling compacts and flushes for region at 1733956864423Disabling writes for close at 1733956864424 (+1 ms)Writing region close event to WAL at 1733956864424Running coprocessor post-close hooks at 1733956864431 (+7 ms)Closed at 1733956864432 (+1 ms) 2024-12-11T22:41:04,432 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1733956859502.2f551d316d038ca1e1403bd05f890572. 2024-12-11T22:41:04,432 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 860d604009b38831efaa4f23b2d17596, disabling compactions & flushes 2024-12-11T22:41:04,432 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1733956859502.860d604009b38831efaa4f23b2d17596. 2024-12-11T22:41:04,432 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1733956859502.860d604009b38831efaa4f23b2d17596. 2024-12-11T22:41:04,432 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1733956859502.860d604009b38831efaa4f23b2d17596. after waiting 0 ms 2024-12-11T22:41:04,432 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1733956859502.860d604009b38831efaa4f23b2d17596. 2024-12-11T22:41:04,436 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/f666d29f964372d8719d169fcecef641/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-12-11T22:41:04,436 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1733956859502.f666d29f964372d8719d169fcecef641. 2024-12-11T22:41:04,437 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f666d29f964372d8719d169fcecef641: Waiting for close lock at 1733956864431Running coprocessor pre-close hooks at 1733956864431Disabling compacts and flushes for region at 1733956864431Disabling writes for close at 1733956864431Writing region close event to WAL at 1733956864431Running coprocessor post-close hooks at 1733956864436 (+5 ms)Closed at 1733956864436 2024-12-11T22:41:04,437 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1733956859502.f666d29f964372d8719d169fcecef641. 2024-12-11T22:41:04,437 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b30c1b01a8241f23347f03f93df7f090, disabling compactions & flushes 2024-12-11T22:41:04,437 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1733956859502.b30c1b01a8241f23347f03f93df7f090. 2024-12-11T22:41:04,437 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1733956859502.b30c1b01a8241f23347f03f93df7f090. 2024-12-11T22:41:04,437 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1733956859502.b30c1b01a8241f23347f03f93df7f090. after waiting 0 ms 2024-12-11T22:41:04,437 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1733956859502.b30c1b01a8241f23347f03f93df7f090. 2024-12-11T22:41:04,437 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/860d604009b38831efaa4f23b2d17596/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-11T22:41:04,438 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1733956859502.860d604009b38831efaa4f23b2d17596. 2024-12-11T22:41:04,438 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 860d604009b38831efaa4f23b2d17596: Waiting for close lock at 1733956864432Running coprocessor pre-close hooks at 1733956864432Disabling compacts and flushes for region at 1733956864432Disabling writes for close at 1733956864432Writing region close event to WAL at 1733956864433 (+1 ms)Running coprocessor post-close hooks at 1733956864438 (+5 ms)Closed at 1733956864438 2024-12-11T22:41:04,438 DEBUG [RS_CLOSE_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/hbase/meta/1588230740/.tmp/info/dab630cdd86e4ee69e0e27d25fb63482 is 138, key is test,fff,1733956859502.419f7368f61981f0a715c20304b74e0f./info:regioninfo/1733956863299/Put/seqid=0 2024-12-11T22:41:04,438 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1733956859502.860d604009b38831efaa4f23b2d17596. 2024-12-11T22:41:04,438 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 419f7368f61981f0a715c20304b74e0f, disabling compactions & flushes 2024-12-11T22:41:04,439 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1733956859502.419f7368f61981f0a715c20304b74e0f. 2024-12-11T22:41:04,439 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1733956859502.419f7368f61981f0a715c20304b74e0f. 2024-12-11T22:41:04,439 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1733956859502.419f7368f61981f0a715c20304b74e0f. after waiting 0 ms 2024-12-11T22:41:04,439 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1733956859502.419f7368f61981f0a715c20304b74e0f. 2024-12-11T22:41:04,445 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/b30c1b01a8241f23347f03f93df7f090/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-11T22:41:04,446 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1733956859502.b30c1b01a8241f23347f03f93df7f090. 2024-12-11T22:41:04,446 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b30c1b01a8241f23347f03f93df7f090: Waiting for close lock at 1733956864437Running coprocessor pre-close hooks at 1733956864437Disabling compacts and flushes for region at 1733956864437Disabling writes for close at 1733956864437Writing region close event to WAL at 1733956864438 (+1 ms)Running coprocessor post-close hooks at 1733956864446 (+8 ms)Closed at 1733956864446 2024-12-11T22:41:04,446 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1733956859502.b30c1b01a8241f23347f03f93df7f090. 2024-12-11T22:41:04,446 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6561ead2eaa9f60da54d4ccff5474a28, disabling compactions & flushes 2024-12-11T22:41:04,446 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1733956859502.6561ead2eaa9f60da54d4ccff5474a28. 2024-12-11T22:41:04,446 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1733956859502.6561ead2eaa9f60da54d4ccff5474a28. 2024-12-11T22:41:04,446 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1733956859502.6561ead2eaa9f60da54d4ccff5474a28. after waiting 0 ms 2024-12-11T22:41:04,446 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1733956859502.6561ead2eaa9f60da54d4ccff5474a28. 2024-12-11T22:41:04,447 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/419f7368f61981f0a715c20304b74e0f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-11T22:41:04,448 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1733956859502.419f7368f61981f0a715c20304b74e0f. 2024-12-11T22:41:04,448 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 419f7368f61981f0a715c20304b74e0f: Waiting for close lock at 1733956864438Running coprocessor pre-close hooks at 1733956864438Disabling compacts and flushes for region at 1733956864438Disabling writes for close at 1733956864439 (+1 ms)Writing region close event to WAL at 1733956864439Running coprocessor post-close hooks at 1733956864448 (+9 ms)Closed at 1733956864448 2024-12-11T22:41:04,448 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1733956859502.419f7368f61981f0a715c20304b74e0f. 2024-12-11T22:41:04,448 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ad4bb3db7f3f05f24b6115fc887d6ab5, disabling compactions & flushes 2024-12-11T22:41:04,448 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1733956859502.ad4bb3db7f3f05f24b6115fc887d6ab5. 2024-12-11T22:41:04,448 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1733956859502.ad4bb3db7f3f05f24b6115fc887d6ab5. 2024-12-11T22:41:04,449 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1733956859502.ad4bb3db7f3f05f24b6115fc887d6ab5. after waiting 0 ms 2024-12-11T22:41:04,449 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1733956859502.ad4bb3db7f3f05f24b6115fc887d6ab5. 2024-12-11T22:41:04,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741869_1045 (size=8177) 2024-12-11T22:41:04,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741869_1045 (size=8177) 2024-12-11T22:41:04,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741869_1045 (size=8177) 2024-12-11T22:41:04,455 INFO [RS_CLOSE_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.64 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/hbase/meta/1588230740/.tmp/info/dab630cdd86e4ee69e0e27d25fb63482 2024-12-11T22:41:04,458 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/ad4bb3db7f3f05f24b6115fc887d6ab5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-11T22:41:04,458 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/6561ead2eaa9f60da54d4ccff5474a28/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-11T22:41:04,458 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1733956859502.ad4bb3db7f3f05f24b6115fc887d6ab5. 2024-12-11T22:41:04,459 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ad4bb3db7f3f05f24b6115fc887d6ab5: Waiting for close lock at 1733956864448Running coprocessor pre-close hooks at 1733956864448Disabling compacts and flushes for region at 1733956864448Disabling writes for close at 1733956864449 (+1 ms)Writing region close event to WAL at 1733956864451 (+2 ms)Running coprocessor post-close hooks at 1733956864458 (+7 ms)Closed at 1733956864458 2024-12-11T22:41:04,459 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1733956859502.ad4bb3db7f3f05f24b6115fc887d6ab5. 2024-12-11T22:41:04,459 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1733956859502.6561ead2eaa9f60da54d4ccff5474a28. 2024-12-11T22:41:04,459 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6561ead2eaa9f60da54d4ccff5474a28: Waiting for close lock at 1733956864446Running coprocessor pre-close hooks at 1733956864446Disabling compacts and flushes for region at 1733956864446Disabling writes for close at 1733956864446Writing region close event to WAL at 1733956864451 (+5 ms)Running coprocessor post-close hooks at 1733956864459 (+8 ms)Closed at 1733956864459 2024-12-11T22:41:04,459 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 66aac9e74e8263b0b8914fdc86eec66c, disabling compactions & flushes 2024-12-11T22:41:04,459 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1733956859502.66aac9e74e8263b0b8914fdc86eec66c. 2024-12-11T22:41:04,459 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1733956859502.6561ead2eaa9f60da54d4ccff5474a28. 2024-12-11T22:41:04,459 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1733956859502.66aac9e74e8263b0b8914fdc86eec66c. 2024-12-11T22:41:04,459 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1733956859502.66aac9e74e8263b0b8914fdc86eec66c. after waiting 0 ms 2024-12-11T22:41:04,459 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1733956859502.66aac9e74e8263b0b8914fdc86eec66c. 2024-12-11T22:41:04,459 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 00fd1046c94e7f507c0ee43e6555d034, disabling compactions & flushes 2024-12-11T22:41:04,459 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1733956859502.00fd1046c94e7f507c0ee43e6555d034. 2024-12-11T22:41:04,459 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1733956859502.00fd1046c94e7f507c0ee43e6555d034. 2024-12-11T22:41:04,459 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1733956859502.00fd1046c94e7f507c0ee43e6555d034. after waiting 0 ms 2024-12-11T22:41:04,459 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1733956859502.00fd1046c94e7f507c0ee43e6555d034. 2024-12-11T22:41:04,464 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/00fd1046c94e7f507c0ee43e6555d034/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-11T22:41:04,464 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/66aac9e74e8263b0b8914fdc86eec66c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-11T22:41:04,465 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1733956859502.00fd1046c94e7f507c0ee43e6555d034. 2024-12-11T22:41:04,465 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 00fd1046c94e7f507c0ee43e6555d034: Waiting for close lock at 1733956864459Running coprocessor pre-close hooks at 1733956864459Disabling compacts and flushes for region at 1733956864459Disabling writes for close at 1733956864459Writing region close event to WAL at 1733956864460 (+1 ms)Running coprocessor post-close hooks at 1733956864465 (+5 ms)Closed at 1733956864465 2024-12-11T22:41:04,465 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1733956859502.00fd1046c94e7f507c0ee43e6555d034. 2024-12-11T22:41:04,465 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1733956859502.66aac9e74e8263b0b8914fdc86eec66c. 2024-12-11T22:41:04,465 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f19f540286846a30911c41a83b97457e, disabling compactions & flushes 2024-12-11T22:41:04,465 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 66aac9e74e8263b0b8914fdc86eec66c: Waiting for close lock at 1733956864459Running coprocessor pre-close hooks at 1733956864459Disabling compacts and flushes for region at 1733956864459Disabling writes for close at 1733956864459Writing region close event to WAL at 1733956864460 (+1 ms)Running coprocessor post-close hooks at 1733956864465 (+5 ms)Closed at 1733956864465 2024-12-11T22:41:04,465 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1733956859502.f19f540286846a30911c41a83b97457e. 2024-12-11T22:41:04,465 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1733956859502.f19f540286846a30911c41a83b97457e. 2024-12-11T22:41:04,465 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1733956859502.f19f540286846a30911c41a83b97457e. after waiting 0 ms 2024-12-11T22:41:04,465 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1733956859502.66aac9e74e8263b0b8914fdc86eec66c. 2024-12-11T22:41:04,465 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1733956859502.f19f540286846a30911c41a83b97457e. 2024-12-11T22:41:04,465 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f219e06826a4a3ede110adacdf28f6f4, disabling compactions & flushes 2024-12-11T22:41:04,466 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1733956859502.f219e06826a4a3ede110adacdf28f6f4. 2024-12-11T22:41:04,466 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1733956859502.f219e06826a4a3ede110adacdf28f6f4. 2024-12-11T22:41:04,466 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1733956859502.f219e06826a4a3ede110adacdf28f6f4. after waiting 0 ms 2024-12-11T22:41:04,466 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1733956859502.f219e06826a4a3ede110adacdf28f6f4. 2024-12-11T22:41:04,466 DEBUG [RS_CLOSE_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/hbase/meta/1588230740/.tmp/info/dab630cdd86e4ee69e0e27d25fb63482 as hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/hbase/meta/1588230740/info/dab630cdd86e4ee69e0e27d25fb63482 2024-12-11T22:41:04,470 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/f19f540286846a30911c41a83b97457e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-11T22:41:04,470 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/f219e06826a4a3ede110adacdf28f6f4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-11T22:41:04,471 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1733956859502.f19f540286846a30911c41a83b97457e. 2024-12-11T22:41:04,471 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1733956859502.f219e06826a4a3ede110adacdf28f6f4. 2024-12-11T22:41:04,471 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f19f540286846a30911c41a83b97457e: Waiting for close lock at 1733956864465Running coprocessor pre-close hooks at 1733956864465Disabling compacts and flushes for region at 1733956864465Disabling writes for close at 1733956864465Writing region close event to WAL at 1733956864466 (+1 ms)Running coprocessor post-close hooks at 1733956864471 (+5 ms)Closed at 1733956864471 2024-12-11T22:41:04,471 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f219e06826a4a3ede110adacdf28f6f4: Waiting for close lock at 1733956864465Running coprocessor pre-close hooks at 1733956864465Disabling compacts and flushes for region at 1733956864465Disabling writes for close at 1733956864466 (+1 ms)Writing region close event to WAL at 1733956864466Running coprocessor post-close hooks at 1733956864471 (+5 ms)Closed at 1733956864471 2024-12-11T22:41:04,471 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1733956859502.f219e06826a4a3ede110adacdf28f6f4. 2024-12-11T22:41:04,471 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1733956859502.f19f540286846a30911c41a83b97457e. 2024-12-11T22:41:04,471 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 445df7fd625f94e0eb0712fcd777ebda, disabling compactions & flushes 2024-12-11T22:41:04,471 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5a93ec512c97cf46e8942b91f4c8f088, disabling compactions & flushes 2024-12-11T22:41:04,471 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1733956859502.445df7fd625f94e0eb0712fcd777ebda. 2024-12-11T22:41:04,471 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1733956859502.5a93ec512c97cf46e8942b91f4c8f088. 2024-12-11T22:41:04,471 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1733956859502.445df7fd625f94e0eb0712fcd777ebda. 2024-12-11T22:41:04,471 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1733956859502.5a93ec512c97cf46e8942b91f4c8f088. 2024-12-11T22:41:04,471 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1733956859502.445df7fd625f94e0eb0712fcd777ebda. after waiting 0 ms 2024-12-11T22:41:04,471 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1733956859502.5a93ec512c97cf46e8942b91f4c8f088. after waiting 0 ms 2024-12-11T22:41:04,471 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1733956859502.445df7fd625f94e0eb0712fcd777ebda. 2024-12-11T22:41:04,471 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1733956859502.5a93ec512c97cf46e8942b91f4c8f088. 2024-12-11T22:41:04,475 INFO [RS_CLOSE_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/hbase/meta/1588230740/info/dab630cdd86e4ee69e0e27d25fb63482, entries=24, sequenceid=75, filesize=8.0 K 2024-12-11T22:41:04,477 INFO [RS_CLOSE_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.64 KB/2708, heapSize ~4.66 KB/4768, currentSize=0 B/0 for 1588230740 in 51ms, sequenceid=75, compaction requested=false 2024-12-11T22:41:04,477 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/445df7fd625f94e0eb0712fcd777ebda/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-11T22:41:04,477 INFO [regionserver/56d7a8405f4a:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-11T22:41:04,477 INFO [regionserver/56d7a8405f4a:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-11T22:41:04,478 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1733956859502.445df7fd625f94e0eb0712fcd777ebda. 2024-12-11T22:41:04,478 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 445df7fd625f94e0eb0712fcd777ebda: Waiting for close lock at 1733956864471Running coprocessor pre-close hooks at 1733956864471Disabling compacts and flushes for region at 1733956864471Disabling writes for close at 1733956864471Writing region close event to WAL at 1733956864472 (+1 ms)Running coprocessor post-close hooks at 1733956864477 (+5 ms)Closed at 1733956864477 2024-12-11T22:41:04,478 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1733956859502.445df7fd625f94e0eb0712fcd777ebda. 2024-12-11T22:41:04,478 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 65f30d8a32c00877e2cf6f22e7f50620, disabling compactions & flushes 2024-12-11T22:41:04,478 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1733956859502.65f30d8a32c00877e2cf6f22e7f50620. 2024-12-11T22:41:04,478 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1733956859502.65f30d8a32c00877e2cf6f22e7f50620. 2024-12-11T22:41:04,478 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1733956859502.65f30d8a32c00877e2cf6f22e7f50620. after waiting 0 ms 2024-12-11T22:41:04,478 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1733956859502.65f30d8a32c00877e2cf6f22e7f50620. 2024-12-11T22:41:04,478 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/5a93ec512c97cf46e8942b91f4c8f088/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-11T22:41:04,479 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1733956859502.5a93ec512c97cf46e8942b91f4c8f088. 2024-12-11T22:41:04,479 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5a93ec512c97cf46e8942b91f4c8f088: Waiting for close lock at 1733956864471Running coprocessor pre-close hooks at 1733956864471Disabling compacts and flushes for region at 1733956864471Disabling writes for close at 1733956864471Writing region close event to WAL at 1733956864473 (+2 ms)Running coprocessor post-close hooks at 1733956864479 (+6 ms)Closed at 1733956864479 2024-12-11T22:41:04,479 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1733956859502.5a93ec512c97cf46e8942b91f4c8f088. 2024-12-11T22:41:04,480 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6801ac4c1cb1416e045d462c592d2dc5, disabling compactions & flushes 2024-12-11T22:41:04,480 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1733956859502.6801ac4c1cb1416e045d462c592d2dc5. 2024-12-11T22:41:04,480 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1733956859502.6801ac4c1cb1416e045d462c592d2dc5. 2024-12-11T22:41:04,480 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1733956859502.6801ac4c1cb1416e045d462c592d2dc5. after waiting 0 ms 2024-12-11T22:41:04,480 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1733956859502.6801ac4c1cb1416e045d462c592d2dc5. 2024-12-11T22:41:04,482 INFO [regionserver/56d7a8405f4a:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-11T22:41:04,482 INFO [regionserver/56d7a8405f4a:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-11T22:41:04,487 DEBUG [RS_CLOSE_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/hbase/meta/1588230740/recovered.edits/78.seqid, newMaxSeqId=78, maxSeqId=64 2024-12-11T22:41:04,487 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/65f30d8a32c00877e2cf6f22e7f50620/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-11T22:41:04,488 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1733956859502.65f30d8a32c00877e2cf6f22e7f50620. 2024-12-11T22:41:04,488 DEBUG [RS_CLOSE_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-11T22:41:04,488 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 65f30d8a32c00877e2cf6f22e7f50620: Waiting for close lock at 1733956864478Running coprocessor pre-close hooks at 1733956864478Disabling compacts and flushes for region at 1733956864478Disabling writes for close at 1733956864478Writing region close event to WAL at 1733956864478Running coprocessor post-close hooks at 1733956864488 (+10 ms)Closed at 1733956864488 2024-12-11T22:41:04,488 INFO [RS_CLOSE_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-11T22:41:04,488 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1733956859502.65f30d8a32c00877e2cf6f22e7f50620. 2024-12-11T22:41:04,488 DEBUG [RS_CLOSE_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733956864425Running coprocessor pre-close hooks at 1733956864425Disabling compacts and flushes for region at 1733956864425Disabling writes for close at 1733956864425Obtaining lock to block concurrent updates at 1733956864425Preparing flush snapshotting stores in 1588230740 at 1733956864425Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2708, getHeapSize=5488, getOffHeapSize=0, getCellsCount=24 at 1733956864431 (+6 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733956864432 (+1 ms)Flushing 1588230740/info: creating writer at 1733956864432Flushing 1588230740/info: appending metadata at 1733956864438 (+6 ms)Flushing 1588230740/info: closing flushed file at 1733956864438Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@187f2090: reopening flushed file at 1733956864465 (+27 ms)Finished flush of dataSize ~2.64 KB/2708, heapSize ~4.66 KB/4768, currentSize=0 B/0 for 1588230740 in 51ms, sequenceid=75, compaction requested=false at 1733956864477 (+12 ms)Writing region close event to WAL at 1733956864483 (+6 ms)Running coprocessor post-close hooks at 1733956864488 (+5 ms)Closed at 1733956864488 2024-12-11T22:41:04,489 DEBUG [RS_CLOSE_META-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-11T22:41:04,489 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9f169feb3190772cec56c0ae51d57c26, disabling compactions & flushes 2024-12-11T22:41:04,489 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1733956859502.9f169feb3190772cec56c0ae51d57c26. 2024-12-11T22:41:04,489 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1733956859502.9f169feb3190772cec56c0ae51d57c26. 2024-12-11T22:41:04,489 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1733956859502.9f169feb3190772cec56c0ae51d57c26. after waiting 0 ms 2024-12-11T22:41:04,489 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1733956859502.9f169feb3190772cec56c0ae51d57c26. 2024-12-11T22:41:04,491 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/6801ac4c1cb1416e045d462c592d2dc5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-11T22:41:04,492 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1733956859502.6801ac4c1cb1416e045d462c592d2dc5. 2024-12-11T22:41:04,492 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6801ac4c1cb1416e045d462c592d2dc5: Waiting for close lock at 1733956864480Running coprocessor pre-close hooks at 1733956864480Disabling compacts and flushes for region at 1733956864480Disabling writes for close at 1733956864480Writing region close event to WAL at 1733956864483 (+3 ms)Running coprocessor post-close hooks at 1733956864492 (+9 ms)Closed at 1733956864492 2024-12-11T22:41:04,492 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1733956859502.6801ac4c1cb1416e045d462c592d2dc5. 2024-12-11T22:41:04,492 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f7dbe11f45ae33a883424947e613c7c6, disabling compactions & flushes 2024-12-11T22:41:04,492 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1733956859502.f7dbe11f45ae33a883424947e613c7c6. 2024-12-11T22:41:04,492 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1733956859502.f7dbe11f45ae33a883424947e613c7c6. 2024-12-11T22:41:04,492 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1733956859502.f7dbe11f45ae33a883424947e613c7c6. after waiting 0 ms 2024-12-11T22:41:04,492 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1733956859502.f7dbe11f45ae33a883424947e613c7c6. 2024-12-11T22:41:04,493 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/9f169feb3190772cec56c0ae51d57c26/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-11T22:41:04,494 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1733956859502.9f169feb3190772cec56c0ae51d57c26. 2024-12-11T22:41:04,494 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9f169feb3190772cec56c0ae51d57c26: Waiting for close lock at 1733956864489Running coprocessor pre-close hooks at 1733956864489Disabling compacts and flushes for region at 1733956864489Disabling writes for close at 1733956864489Writing region close event to WAL at 1733956864489Running coprocessor post-close hooks at 1733956864494 (+5 ms)Closed at 1733956864494 2024-12-11T22:41:04,494 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1733956859502.9f169feb3190772cec56c0ae51d57c26. 2024-12-11T22:41:04,494 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing da518288db554a985096494f24adc59f, disabling compactions & flushes 2024-12-11T22:41:04,494 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1733956859502.da518288db554a985096494f24adc59f. 2024-12-11T22:41:04,494 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1733956859502.da518288db554a985096494f24adc59f. 2024-12-11T22:41:04,494 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1733956859502.da518288db554a985096494f24adc59f. after waiting 0 ms 2024-12-11T22:41:04,494 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1733956859502.da518288db554a985096494f24adc59f. 2024-12-11T22:41:04,497 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/f7dbe11f45ae33a883424947e613c7c6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-11T22:41:04,498 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1733956859502.f7dbe11f45ae33a883424947e613c7c6. 2024-12-11T22:41:04,498 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f7dbe11f45ae33a883424947e613c7c6: Waiting for close lock at 1733956864492Running coprocessor pre-close hooks at 1733956864492Disabling compacts and flushes for region at 1733956864492Disabling writes for close at 1733956864492Writing region close event to WAL at 1733956864493 (+1 ms)Running coprocessor post-close hooks at 1733956864498 (+5 ms)Closed at 1733956864498 2024-12-11T22:41:04,498 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1733956859502.f7dbe11f45ae33a883424947e613c7c6. 2024-12-11T22:41:04,498 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/da518288db554a985096494f24adc59f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-11T22:41:04,498 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 01df3c1c862de5abd063d235e117a115, disabling compactions & flushes 2024-12-11T22:41:04,498 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ppp,1733956859502.01df3c1c862de5abd063d235e117a115. 2024-12-11T22:41:04,498 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1733956859502.01df3c1c862de5abd063d235e117a115. 2024-12-11T22:41:04,498 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1733956859502.01df3c1c862de5abd063d235e117a115. after waiting 0 ms 2024-12-11T22:41:04,498 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1733956859502.01df3c1c862de5abd063d235e117a115. 2024-12-11T22:41:04,499 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1733956859502.da518288db554a985096494f24adc59f. 2024-12-11T22:41:04,499 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for da518288db554a985096494f24adc59f: Waiting for close lock at 1733956864494Running coprocessor pre-close hooks at 1733956864494Disabling compacts and flushes for region at 1733956864494Disabling writes for close at 1733956864494Writing region close event to WAL at 1733956864495 (+1 ms)Running coprocessor post-close hooks at 1733956864499 (+4 ms)Closed at 1733956864499 2024-12-11T22:41:04,499 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1733956859502.da518288db554a985096494f24adc59f. 2024-12-11T22:41:04,499 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a0e02a7a4f8d57169d1f435eabc00839, disabling compactions & flushes 2024-12-11T22:41:04,499 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1733956859502.a0e02a7a4f8d57169d1f435eabc00839. 2024-12-11T22:41:04,499 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1733956859502.a0e02a7a4f8d57169d1f435eabc00839. 2024-12-11T22:41:04,499 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1733956859502.a0e02a7a4f8d57169d1f435eabc00839. after waiting 0 ms 2024-12-11T22:41:04,499 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1733956859502.a0e02a7a4f8d57169d1f435eabc00839. 2024-12-11T22:41:04,503 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/01df3c1c862de5abd063d235e117a115/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-11T22:41:04,503 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/a0e02a7a4f8d57169d1f435eabc00839/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-11T22:41:04,504 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ppp,1733956859502.01df3c1c862de5abd063d235e117a115. 2024-12-11T22:41:04,504 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1733956859502.a0e02a7a4f8d57169d1f435eabc00839. 2024-12-11T22:41:04,504 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a0e02a7a4f8d57169d1f435eabc00839: Waiting for close lock at 1733956864499Running coprocessor pre-close hooks at 1733956864499Disabling compacts and flushes for region at 1733956864499Disabling writes for close at 1733956864499Writing region close event to WAL at 1733956864500 (+1 ms)Running coprocessor post-close hooks at 1733956864504 (+4 ms)Closed at 1733956864504 2024-12-11T22:41:04,504 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 01df3c1c862de5abd063d235e117a115: Waiting for close lock at 1733956864498Running coprocessor pre-close hooks at 1733956864498Disabling compacts and flushes for region at 1733956864498Disabling writes for close at 1733956864498Writing region close event to WAL at 1733956864499 (+1 ms)Running coprocessor post-close hooks at 1733956864504 (+5 ms)Closed at 1733956864504 2024-12-11T22:41:04,504 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1733956859502.a0e02a7a4f8d57169d1f435eabc00839. 2024-12-11T22:41:04,504 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ppp,1733956859502.01df3c1c862de5abd063d235e117a115. 2024-12-11T22:41:04,504 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ff9e1faa3192d62c84407efca19e0a89, disabling compactions & flushes 2024-12-11T22:41:04,504 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing de3803c768bc063bdc71e495af58eb48, disabling compactions & flushes 2024-12-11T22:41:04,504 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1733956859502.de3803c768bc063bdc71e495af58eb48. 2024-12-11T22:41:04,504 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1733956859502.ff9e1faa3192d62c84407efca19e0a89. 2024-12-11T22:41:04,504 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1733956859502.ff9e1faa3192d62c84407efca19e0a89. 2024-12-11T22:41:04,504 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1733956859502.de3803c768bc063bdc71e495af58eb48. 2024-12-11T22:41:04,504 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1733956859502.ff9e1faa3192d62c84407efca19e0a89. after waiting 0 ms 2024-12-11T22:41:04,504 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1733956859502.de3803c768bc063bdc71e495af58eb48. after waiting 0 ms 2024-12-11T22:41:04,504 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1733956859502.de3803c768bc063bdc71e495af58eb48. 2024-12-11T22:41:04,504 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1733956859502.ff9e1faa3192d62c84407efca19e0a89. 2024-12-11T22:41:04,509 INFO [regionserver/56d7a8405f4a:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-11T22:41:04,509 INFO [regionserver/56d7a8405f4a:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-11T22:41:04,509 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/de3803c768bc063bdc71e495af58eb48/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-11T22:41:04,509 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/ff9e1faa3192d62c84407efca19e0a89/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-11T22:41:04,509 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1733956859502.de3803c768bc063bdc71e495af58eb48. 2024-12-11T22:41:04,510 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for de3803c768bc063bdc71e495af58eb48: Waiting for close lock at 1733956864504Running coprocessor pre-close hooks at 1733956864504Disabling compacts and flushes for region at 1733956864504Disabling writes for close at 1733956864504Writing region close event to WAL at 1733956864505 (+1 ms)Running coprocessor post-close hooks at 1733956864509 (+4 ms)Closed at 1733956864509 2024-12-11T22:41:04,510 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1733956859502.de3803c768bc063bdc71e495af58eb48. 2024-12-11T22:41:04,510 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1733956859502.ff9e1faa3192d62c84407efca19e0a89. 2024-12-11T22:41:04,510 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b1a38f11b48b53983535b863fd4392ff, disabling compactions & flushes 2024-12-11T22:41:04,510 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1733956859502.b1a38f11b48b53983535b863fd4392ff. 2024-12-11T22:41:04,510 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ff9e1faa3192d62c84407efca19e0a89: Waiting for close lock at 1733956864504Running coprocessor pre-close hooks at 1733956864504Disabling compacts and flushes for region at 1733956864504Disabling writes for close at 1733956864504Writing region close event to WAL at 1733956864505 (+1 ms)Running coprocessor post-close hooks at 1733956864510 (+5 ms)Closed at 1733956864510 2024-12-11T22:41:04,510 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1733956859502.b1a38f11b48b53983535b863fd4392ff. 2024-12-11T22:41:04,510 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1733956859502.b1a38f11b48b53983535b863fd4392ff. after waiting 0 ms 2024-12-11T22:41:04,510 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1733956859502.b1a38f11b48b53983535b863fd4392ff. 2024-12-11T22:41:04,510 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1733956859502.ff9e1faa3192d62c84407efca19e0a89. 2024-12-11T22:41:04,510 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b31ee44b6609c355adc37986b13d239c, disabling compactions & flushes 2024-12-11T22:41:04,510 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1733956859502.b31ee44b6609c355adc37986b13d239c. 2024-12-11T22:41:04,510 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1733956859502.b31ee44b6609c355adc37986b13d239c. 2024-12-11T22:41:04,510 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1733956859502.b31ee44b6609c355adc37986b13d239c. after waiting 0 ms 2024-12-11T22:41:04,510 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1733956859502.b31ee44b6609c355adc37986b13d239c. 2024-12-11T22:41:04,514 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/b1a38f11b48b53983535b863fd4392ff/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-11T22:41:04,514 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/b31ee44b6609c355adc37986b13d239c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-11T22:41:04,515 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1733956859502.b1a38f11b48b53983535b863fd4392ff. 2024-12-11T22:41:04,515 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1733956859502.b31ee44b6609c355adc37986b13d239c. 2024-12-11T22:41:04,515 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b1a38f11b48b53983535b863fd4392ff: Waiting for close lock at 1733956864510Running coprocessor pre-close hooks at 1733956864510Disabling compacts and flushes for region at 1733956864510Disabling writes for close at 1733956864510Writing region close event to WAL at 1733956864510Running coprocessor post-close hooks at 1733956864515 (+5 ms)Closed at 1733956864515 2024-12-11T22:41:04,515 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b31ee44b6609c355adc37986b13d239c: Waiting for close lock at 1733956864510Running coprocessor pre-close hooks at 1733956864510Disabling compacts and flushes for region at 1733956864510Disabling writes for close at 1733956864510Writing region close event to WAL at 1733956864511 (+1 ms)Running coprocessor post-close hooks at 1733956864515 (+4 ms)Closed at 1733956864515 2024-12-11T22:41:04,515 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1733956859502.b1a38f11b48b53983535b863fd4392ff. 2024-12-11T22:41:04,515 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1733956859502.b31ee44b6609c355adc37986b13d239c. 2024-12-11T22:41:04,515 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e9e94fba4d03035b5feaab607eb13853, disabling compactions & flushes 2024-12-11T22:41:04,515 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1733956859502.e9e94fba4d03035b5feaab607eb13853. 2024-12-11T22:41:04,515 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1733956859502.e9e94fba4d03035b5feaab607eb13853. 2024-12-11T22:41:04,515 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1733956859502.e9e94fba4d03035b5feaab607eb13853. after waiting 0 ms 2024-12-11T22:41:04,515 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1733956859502.e9e94fba4d03035b5feaab607eb13853. 2024-12-11T22:41:04,520 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/data/default/test/e9e94fba4d03035b5feaab607eb13853/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-11T22:41:04,520 INFO [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1733956859502.e9e94fba4d03035b5feaab607eb13853. 2024-12-11T22:41:04,520 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e9e94fba4d03035b5feaab607eb13853: Waiting for close lock at 1733956864515Running coprocessor pre-close hooks at 1733956864515Disabling compacts and flushes for region at 1733956864515Disabling writes for close at 1733956864515Writing region close event to WAL at 1733956864516 (+1 ms)Running coprocessor post-close hooks at 1733956864520 (+4 ms)Closed at 1733956864520 2024-12-11T22:41:04,520 DEBUG [RS_CLOSE_REGION-regionserver/56d7a8405f4a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1733956859502.e9e94fba4d03035b5feaab607eb13853. 2024-12-11T22:41:04,625 INFO [RS:1;56d7a8405f4a:40421 {}] regionserver.HRegionServer(976): stopping server 56d7a8405f4a,40421,1733956857131; all regions closed. 2024-12-11T22:41:04,625 INFO [RS:0;56d7a8405f4a:34703 {}] regionserver.HRegionServer(976): stopping server 56d7a8405f4a,34703,1733956857011; all regions closed. 2024-12-11T22:41:04,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741865_1041 (size=4192) 2024-12-11T22:41:04,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741865_1041 (size=4192) 2024-12-11T22:41:04,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741865_1041 (size=4192) 2024-12-11T22:41:04,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741833_1009 (size=7074) 2024-12-11T22:41:04,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741833_1009 (size=7074) 2024-12-11T22:41:04,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741833_1009 (size=7074) 2024-12-11T22:41:04,637 DEBUG [RS:1;56d7a8405f4a:40421 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/oldWALs 2024-12-11T22:41:04,637 DEBUG [RS:0;56d7a8405f4a:34703 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/oldWALs 2024-12-11T22:41:04,637 INFO [RS:0;56d7a8405f4a:34703 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 56d7a8405f4a%2C34703%2C1733956857011.meta:.meta(num 1733956862595) 2024-12-11T22:41:04,637 INFO [RS:1;56d7a8405f4a:40421 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 56d7a8405f4a%2C40421%2C1733956857131:(num 1733956858632) 2024-12-11T22:41:04,637 DEBUG [RS:1;56d7a8405f4a:40421 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T22:41:04,638 INFO [RS:1;56d7a8405f4a:40421 {}] regionserver.LeaseManager(133): Closed leases 2024-12-11T22:41:04,638 INFO [RS:1;56d7a8405f4a:40421 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-11T22:41:04,638 INFO [RS:1;56d7a8405f4a:40421 {}] hbase.ChoreService(370): Chore service for: regionserver/56d7a8405f4a:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-11T22:41:04,638 INFO [RS:1;56d7a8405f4a:40421 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-11T22:41:04,638 INFO [RS:1;56d7a8405f4a:40421 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-11T22:41:04,638 INFO [regionserver/56d7a8405f4a:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-11T22:41:04,638 INFO [RS:1;56d7a8405f4a:40421 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-11T22:41:04,638 INFO [RS:1;56d7a8405f4a:40421 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-11T22:41:04,638 INFO [RS:1;56d7a8405f4a:40421 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40421 2024-12-11T22:41:04,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741834_1010 (size=7064) 2024-12-11T22:41:04,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741834_1010 (size=7064) 2024-12-11T22:41:04,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741834_1010 (size=7064) 2024-12-11T22:41:04,643 DEBUG [RS:0;56d7a8405f4a:34703 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/oldWALs 2024-12-11T22:41:04,643 INFO [RS:0;56d7a8405f4a:34703 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 56d7a8405f4a%2C34703%2C1733956857011:(num 1733956858632) 2024-12-11T22:41:04,643 DEBUG [RS:0;56d7a8405f4a:34703 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T22:41:04,643 INFO [RS:0;56d7a8405f4a:34703 {}] regionserver.LeaseManager(133): Closed leases 2024-12-11T22:41:04,643 INFO [RS:0;56d7a8405f4a:34703 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-11T22:41:04,643 INFO [RS:0;56d7a8405f4a:34703 {}] hbase.ChoreService(370): Chore service for: regionserver/56d7a8405f4a:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-11T22:41:04,643 INFO [RS:0;56d7a8405f4a:34703 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-11T22:41:04,643 INFO [regionserver/56d7a8405f4a:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-11T22:41:04,644 INFO [RS:0;56d7a8405f4a:34703 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:34703 2024-12-11T22:41:04,653 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T22:41:04,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40421-0x1001717a55e0002, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/56d7a8405f4a,40421,1733956857131 2024-12-11T22:41:04,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-11T22:41:04,670 INFO [RS:1;56d7a8405f4a:40421 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-11T22:41:04,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34703-0x1001717a55e0001, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/56d7a8405f4a,34703,1733956857011 2024-12-11T22:41:04,678 INFO [RS:0;56d7a8405f4a:34703 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-11T22:41:04,678 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [56d7a8405f4a,40421,1733956857131] 2024-12-11T22:41:04,695 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/56d7a8405f4a,40421,1733956857131 already deleted, retry=false 2024-12-11T22:41:04,695 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 56d7a8405f4a,40421,1733956857131 expired; onlineServers=1 2024-12-11T22:41:04,695 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [56d7a8405f4a,34703,1733956857011] 2024-12-11T22:41:04,697 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-12-11T22:41:04,703 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/56d7a8405f4a,34703,1733956857011 already deleted, retry=false 2024-12-11T22:41:04,703 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 56d7a8405f4a,34703,1733956857011 expired; onlineServers=0 2024-12-11T22:41:04,704 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '56d7a8405f4a,34843,1733956856287' ***** 2024-12-11T22:41:04,704 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-11T22:41:04,704 INFO [M:0;56d7a8405f4a:34843 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-11T22:41:04,704 INFO [M:0;56d7a8405f4a:34843 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-11T22:41:04,704 DEBUG [M:0;56d7a8405f4a:34843 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-11T22:41:04,704 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-11T22:41:04,704 DEBUG [M:0;56d7a8405f4a:34843 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-11T22:41:04,704 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster-HFileCleaner.large.0-1733956858266 {}] cleaner.HFileCleaner(306): Exit Thread[master/56d7a8405f4a:0:becomeActiveMaster-HFileCleaner.large.0-1733956858266,5,FailOnTimeoutGroup] 2024-12-11T22:41:04,704 DEBUG [master/56d7a8405f4a:0:becomeActiveMaster-HFileCleaner.small.0-1733956858266 {}] cleaner.HFileCleaner(306): Exit Thread[master/56d7a8405f4a:0:becomeActiveMaster-HFileCleaner.small.0-1733956858266,5,FailOnTimeoutGroup] 2024-12-11T22:41:04,704 INFO [M:0;56d7a8405f4a:34843 {}] hbase.ChoreService(370): Chore service for: master/56d7a8405f4a:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-11T22:41:04,704 INFO [M:0;56d7a8405f4a:34843 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-11T22:41:04,705 DEBUG [M:0;56d7a8405f4a:34843 {}] master.HMaster(1795): Stopping service threads 2024-12-11T22:41:04,705 INFO [M:0;56d7a8405f4a:34843 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-11T22:41:04,705 INFO [M:0;56d7a8405f4a:34843 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:41:04,705 INFO [M:0;56d7a8405f4a:34843 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-11T22:41:04,705 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-11T22:41:04,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-11T22:41:04,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:41:04,712 DEBUG [M:0;56d7a8405f4a:34843 {}] zookeeper.ZKUtil(347): master:34843-0x1001717a55e0000, quorum=127.0.0.1:53529, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-11T22:41:04,712 WARN [M:0;56d7a8405f4a:34843 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-11T22:41:04,713 INFO [M:0;56d7a8405f4a:34843 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/.lastflushedseqids 2024-12-11T22:41:04,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741870_1046 (size=1832) 2024-12-11T22:41:04,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741870_1046 (size=1832) 2024-12-11T22:41:04,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741870_1046 (size=1832) 2024-12-11T22:41:04,725 INFO [M:0;56d7a8405f4a:34843 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-11T22:41:04,725 INFO [M:0;56d7a8405f4a:34843 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-11T22:41:04,725 DEBUG [M:0;56d7a8405f4a:34843 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-11T22:41:04,725 INFO [M:0;56d7a8405f4a:34843 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T22:41:04,725 DEBUG [M:0;56d7a8405f4a:34843 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T22:41:04,725 DEBUG [M:0;56d7a8405f4a:34843 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-11T22:41:04,725 DEBUG [M:0;56d7a8405f4a:34843 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T22:41:04,725 INFO [M:0;56d7a8405f4a:34843 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=260.80 KB heapSize=317.02 KB 2024-12-11T22:41:04,741 DEBUG [M:0;56d7a8405f4a:34843 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4bea6ffe497546d5bf60e9cb2c2d1f76 is 82, key is hbase:meta,,1/info:regioninfo/1733956862903/Put/seqid=0 2024-12-11T22:41:04,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33033 is added to blk_1073741871_1047 (size=6071) 2024-12-11T22:41:04,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741871_1047 (size=6071) 2024-12-11T22:41:04,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36811 is added to blk_1073741871_1047 (size=6071) 2024-12-11T22:41:04,747 INFO [M:0;56d7a8405f4a:34843 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=603 (bloomFilter=true), to=hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4bea6ffe497546d5bf60e9cb2c2d1f76 2024-12-11T22:41:04,769 DEBUG [M:0;56d7a8405f4a:34843 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43559/user/jenkins/test-data/4e467898-7d18-fad5-7b32-1bae3681c28c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/833823a071d046d7b48dcfd5c8ba33b0 is 2987, key is \x00\x00\x00\x00\x00\x00\x00